I found a solution to this myself. It returns the default post type posts instead of excluding the custom post type which is fine for my needs. I updated the code above.

I found a solution to this myself. It returns the default post type posts instead of excluding the custom post type which is fine for my needs.

function my_breakfast_query ( $query ) {
// not an admin page and is the main query
if (!is_admin() && $query->is_main_query()){
if (is_tax( 'food', 'breakfast' )){
  $tax_query = array(
    'relation' => 'OR',
     array(
        'taxonomy' => 'category',
        'field' => 'id',
        'terms' => array( 366 )
     ),
     array(
        'taxonomy' => 'food',
        'field' => 'id',
        'terms' => array( 364 )
     )
  );
  $query->set('post_type','post'); // Added this line to get a working solution
  $query->set('tax_query', $tax_query);
 }
 return $query; // Added this line to get a working solution
 }
}
add_action( 'pre_get_posts', 'my_breakfast_query' );