I was lucky, the theme used it's own function, so it was easy to override. Here is what I end up using

function my_get_the_posts_navigation( $args = array() ) {
    $limit = 5;
    $navigation = '';
 
    // Don't print empty markup if there's only one page.
    if ( $GLOBALS['wp_query']->max_num_pages > 1 ) {
        $args = wp_parse_args( $args, array(
            'prev_text'          => __( 'Older posts' ),
            'next_text'          => __( 'Newer posts' ),
            'screen_reader_text' => __( 'Posts navigation' ),
        ) );
        
 
        $next_link = get_previous_posts_link( $args['next_text'] );
        
        $p = (get_query_var('paged')) ? get_query_var('paged') : 1;
        if ($p < $limit) {
          $prev_link = get_next_posts_link( $args['prev_text'] );
        } else {
          $prev_link = false;
        }
 
        if ( $prev_link ) {
            $navigation .= '<div class="nav-previous">' . $prev_link . '</div>';
        }
 
        if ( $next_link ) {
            $navigation .= '<div class="nav-next">' . $next_link . '</div>';
        }
 
        $navigation = _navigation_markup( $navigation, 'posts-navigation', $args['screen_reader_text'] );
    }
 
    return $navigation;
}

function my_the_posts_navigation( $args = array() ) {
    echo my_get_the_posts_navigation( $args );
}

function my_posts_navigation( $args = array() ) {
  my_the_posts_navigation( $args );
}

// now override the theme pagination function
if ( ! function_exists( 'cactus_paging_nav' ) ) :
function cactus_paging_nav() {
    
    my_the_posts_navigation();
}
endif;

I was lucky, the theme used it's own function, so it was easy to override. Inspired by worpdress native get_the_posts_navigation (wp-includes/link-template.php) here is what I end up using:

function my_get_the_posts_navigation( $args = array() ) {
    $limit = 5;
    $navigation = '';
 
    // Don't print empty markup if there's only one page.
    if ( $GLOBALS['wp_query']->max_num_pages > 1 ) {
        $args = wp_parse_args( $args, array(
            'prev_text'          => __( 'Older posts' ),
            'next_text'          => __( 'Newer posts' ),
            'screen_reader_text' => __( 'Posts navigation' ),
        ) );
        
 
        $next_link = get_previous_posts_link( $args['next_text'] );
        
        $p = (get_query_var('paged')) ? get_query_var('paged') : 1;
        if ($p < $limit) {
          $prev_link = get_next_posts_link( $args['prev_text'] );
        } else {
          $prev_link = false;
        }
 
        if ( $prev_link ) {
            $navigation .= '<div class="nav-previous">' . $prev_link . '</div>';
        }
 
        if ( $next_link ) {
            $navigation .= '<div class="nav-next">' . $next_link . '</div>';
        }
 
        $navigation = _navigation_markup( $navigation, 'posts-navigation', $args['screen_reader_text'] );
    }
 
    return $navigation;
}

function my_the_posts_navigation( $args = array() ) {
    echo my_get_the_posts_navigation( $args );
}

// now override the theme pagination function
if ( ! function_exists( 'cactus_paging_nav' ) ) :
function cactus_paging_nav() {
    
    my_the_posts_navigation();
}
endif;