So, I had to take the set_tax_query_field_to_id action and amend the tax_query after it had been created.

Here is the full Class that I am using to add filters for my custom taxonomies to my various custom post types -

class FGW_CPT_Taxonomy_Filter{
    
    /**
     * The custom post type to alter and the taxonomies to show filters for
     *
     * @var array
     */
    private $cpt = array();
    
    /**
     * Constructor
     *
     * @param required array $cpt   The custom post type and associated taxonomies to show filters for (as "array($cpt => array(tax1, tax2))")
     */
    public function __construct($cpt = array()){
    
         $this->cpt = $cpt;
        add_action('restrict_manage_posts', array($this, 'render_filter'));
        add_action('parse_tax_query', array($this, 'set_tax_query_field_to_id'));
        
    }
    
     /**
     * Render the taxonomy filter
     */
    public function render_filter(){
    
        global $typenow;
        
        $types = array_keys($this->cpt);    // Grab the post types that are being edited
        
        if(!in_array($typenow, $types))     // Check to see if the page now is one of the custom post types supplied by the user
            return false;
        
        $filters = $this->cpt[$typenow];    // Grab all of the taxonomies that shold have a filter
        
        foreach($filters as $tax_slug) :
        
            $tax_obj = get_taxonomy($tax_slug); // Grab the taxonomy object (from the taxonomy slug)
            
            $dropdown_args = array(             // Output the filter dropdown
                'show_option_all'   =>  $tax_obj->labels->all_items,
                'taxonomy'          =>  $tax_slug,
                'name'              =>  $tax_slug,
                'orderby'           =>  'name',
                'selected'          =>  (isset($_GET[$tax_slug]) ? $_GET[strtolower($tax_slug)] : null),
                'hierarchical'      =>  true,
                'depth'             =>  0,
                'show_count'        =>  false,
                'hide_empty'        =>  false
            );
            wp_dropdown_categories($dropdown_args);
            
        endforeach;
        
    }
    
    /**
     * Change the 'field' key in all tax_query related settings to 'id'
     */
    public function set_tax_query_field_to_id($query){

        global $typenow;
        
        if(!isset($query->tax_query))
            return false;
            
        $types = array_keys($this->cpt);    // Grab the post types that are being edited
            
        if(!in_array($typenow, $types))     // Check to see if the page now is one of the custom post types supplied by the user
            return false;
        
        $filters = $this->cpt[$typenow];    // Grab all of the taxonomies that shold have a filter
        
        foreach($filters as $tax_slug) :
        
            if(isset($query->tax_query->queries)) : foreach($query->tax_query->queries as &$single_tax_query) :
            
                    if($single_tax_query['taxonomy'] === $tax_slug)
                        $single_tax_query['field'] = 'id';
                    
                endforeach;
            
            endif;
            
            if(isset($query->tax_query->queried_terms)) : foreach($query->tax_query->queried_terms as $term => &$settings) :
            
                    if($term === $tax_slug)
                        $settings['field'] = 'id';
                    
                endforeach;
            
            endif;
            
        endforeach;
        
    }
    
}

So, I had to make a custom walker and use that with the wp_dropdown_categories() function, as below -

/**
 * Custom extended FGW_Cat_Slug_Walker Class
 * Outputs the term 'slug' as the value of each option, not 'term_id' (as is default)
 */
class FGW_Cat_Slug_Walker extends Walker_CategoryDropdown{

    /**
     * Start the element output
     *
     * @param required string $output   Passed by reference. Used to append additional content
     * @param required object $category Category data object
     * @param required int    $depth    Depth of category in reference to parents. Default 0
     * @param required array  $args     An array of arguments
     * @param required int    $id       ID of the current category
     */
    public function start_el(&$output, $category, $depth = 0, $args = array(), $id = 0){
    
        $pad = str_repeat(' ', $depth * 3);    // Create the padding (before nested terms)
        
        /** Generate the HTML for this option */
        $output.= sprintf("\t".
            '<option class="%1$s" value="%2$s" %3$s>%4$s%5$s</option>',
            /** %1$s - 'class' attribute */     'level-' . $depth,
            /** %2$s - 'value' attribute */     $category->slug,
            /** %3$s - 'selected' attribute */  ($category->slug == $args['selected']) ? ' selected="selected"' : '',
            /** %4$s - option text */           $category->name,
            /** %5$s - The term count */        ($args['show_count']) ? '&nbsp;&nbsp;(' . $category->count . ')' : ''
        );
        
    }
}