Filter 'wp_nav_menu_objects' would help:

add_filter( 'wp_nav_menu_objects', 'remove_sub_items', 10, 2 );
function remove_sub_items( $items,$args ) {
    $new_items = array();
    for ($i=1;$i<count($items)+1;$i++){

        //is lvl0
        if(empty($items[$i]->menu_item_parent)){
           $new_items= array_merge($new_items, nav_tree($items[$i],$items));
        }

    } 
  // var_dump($new_items); die();
    if( $args->theme_location == 'primary' )
        return $new_items; 
    return $items;   
}

function nav_tree($parent,$items){
      $rtn = array();
      $rtn[] = $parent;
      if($parent->my_custom_menu != 'none') return $rtn;
      for ($i=1;$i<count($items)+1;$i++){
         if($items[$i]->menu_item_parent && $items[$i]->menu_item_parent == $parent->ID)
        {
            $rtn= array_merge($rtn,nav_tree($items[$i],$items));

        }

      }
      return $rtn;
}

Filter 'wp_nav_menu_objects' would help:

add_filter( 'wp_nav_menu_objects', 'remove_sub_items', 10, 2 );
function remove_sub_items( $items,$args ) {
    $new_items = array();
    for ($i=1;$i<count($items)+1;$i++){

        //is lvl0
        if(empty($items[$i]->menu_item_parent)){
           $new_items= array_merge($new_items, nav_tree($items[$i],$items));
        }

    } 
  // var_dump($new_items); die();
    if( $args->theme_location == 'primary' )
        return $new_items; 
    return $items;   
}

function nav_tree($parent,$items){
      $rtn = array();
      $rtn[] = $parent;
      //Edit this conditional, return menu level 0 if has custom menu
      if($parent->my_custom_menu && $parent->my_custom_menu !== 'none') return $rtn;
      for ($i=1;$i<count($items)+1;$i++){
         if($items[$i]->menu_item_parent && $items[$i]->menu_item_parent == $parent->ID)
        {
            $rtn= array_merge($rtn,nav_tree($items[$i],$items));

        }

      }
      return $rtn;
}

Filter 'wp_nav_menu_objects' would help:

add_filter( 'wp_nav_menu_objects', 'remove_sub_items' );
function remove_sub_items( $items ) {
    $new_items = array();
    for ($i=0;$i<count($items);$i++){

        if($items[$i]->menu_item_parent && $items[$i]->menu_item_parent > 0)
        {
            for ($j=0;$j<count($items);$j++){
                if($items[$j]->ID == $items[$i]->menu_item_parent && $items[$j]->my_custom_menu !== 'none'){
                    $new_items[]=$items[$i];    
                }
            }
        }
        else{
            $new_items[]=$items[$i];
        }

    } 
    //var_dump($new_items); die();
    return $new_items;    
}

Filter 'wp_nav_menu_objects' would help:

add_filter( 'wp_nav_menu_objects', 'remove_sub_items', 10, 2 );
function remove_sub_items( $items,$args ) {
    $new_items = array();
    for ($i=1;$i<count($items)+1;$i++){

        //is lvl0
        if(empty($items[$i]->menu_item_parent)){
           $new_items= array_merge($new_items, nav_tree($items[$i],$items));
        }

    }  
  // var_dump($new_items); die();
    if( $args->theme_location == 'primary' )
        return $new_items;  
    return $items;   
}

function nav_tree($parent,$items){
      $rtn = array();
      $rtn[] = $parent;
      if($parent->my_custom_menu != 'none') return $rtn;
      for ($i=1;$i<count($items)+1;$i++){
         if($items[$i]->menu_item_parent && $items[$i]->menu_item_parent == $parent->ID)
        {
            $rtn= array_merge($rtn,nav_tree($items[$i],$items));

        }

      }
      return $rtn;
}