I was lucky, the theme used it's own function, so it was easy to override.
Inspired by worpdress native get_the_posts_navigation (wp-includes/link-template.php) here is what I end up using:

    function my_get_the_posts_navigation( $args = array() ) {
        $limit = 5;
        $navigation = '';
     
        // Don't print empty markup if there's only one page.
        if ( $GLOBALS['wp_query']->max_num_pages > 1 ) {
            $args = wp_parse_args( $args, array(
                'prev_text'          => __( 'Older posts' ),
                'next_text'          => __( 'Newer posts' ),
                'screen_reader_text' => __( 'Posts navigation' ),
            ) );
            
     
            $next_link = get_previous_posts_link( $args['next_text'] );
            
            $p = (get_query_var('paged')) ? get_query_var('paged') : 1;
            if ($p < $limit) {
              $prev_link = get_next_posts_link( $args['prev_text'] );
            } else {
              $prev_link = false;
            }
     
            if ( $prev_link ) {
                $navigation .= '<div class="nav-previous">' . $prev_link . '</div>';
            }
     
            if ( $next_link ) {
                $navigation .= '<div class="nav-next">' . $next_link . '</div>';
            }
     
            $navigation = _navigation_markup( $navigation, 'posts-navigation', $args['screen_reader_text'] );
        }
     
        return $navigation;
    }
    
    function my_the_posts_navigation( $args = array() ) {
        echo my_get_the_posts_navigation( $args );
    }

    // now override the theme pagination function
    if ( ! function_exists( 'cactus_paging_nav' ) ) :
    function cactus_paging_nav() {
    	
    	my_the_posts_navigation();
    }
    endif;