I managed to get this working using nothing but WordPress filters, thanks to @Milo's hint.

I created 4 filters:

 - 1 to modify `JOIN` (to add `wp_postmeta`)
 - 1 to modify `WHERE` for the Previous link
 - 1 to modify `WHERE` for the Next link
 - 1 to modify `SORT`

Here's what I came up with, it seems to be working but if anyone spots any issues I would love feedback:

    function get_adjacent_past_events_join($join) {
      if(is_singular('event')) {
        global $wpdb;
        $new_join = $join."INNER JOIN $wpdb->postmeta AS m ON p.ID = m.post_id ";
        return $new_join;
      }
      return $join;
    }
    add_filter('get_previous_post_join', 'get_adjacent_past_events_join');
    add_filter('get_next_post_join', 'get_adjacent_past_events_join');
    
    function get_prev_past_events_where($where) {
      if(is_singular('event')) {
        global $wpdb, $post;
        $id = $post->ID;
        $current_event_date = get_field('date', $id);
        $today = date('Ymd');
        $new_where = "WHERE (m.meta_key = 'date' AND (m.meta_key = 'date' AND CAST(m.meta_value AS CHAR) < '$current_event_date')) AND p.post_type = 'event' AND p.post_status = 'publish' AND (m.meta_key = 'date' AND (m.meta_key = 'date' AND CAST(m.meta_value AS CHAR) < '$today'))";
        return $new_where;
      }
      return $where;
    }
    add_filter('get_previous_post_where', 'get_prev_past_events_where');
    
    function get_next_past_events_where($where) {
      if(is_singular('event')) {
        global $wpdb, $post;
        $id = $post->ID;
        $current_event_date = get_field('date', $id);
        $today = date('Ymd');
        $new_where = "WHERE (m.meta_key = 'date' AND (m.meta_key = 'date' AND CAST(m.meta_value AS CHAR) > '$current_event_date')) AND p.post_type = 'event' AND p.post_status = 'publish' AND (m.meta_key = 'date' AND (m.meta_key = 'date' AND CAST(m.meta_value AS CHAR) < '$today'))";
        return $new_where;
      }
      return $where;
    }
    add_filter('get_next_post_where', 'get_next_past_events_where');
    
    function get_adjacent_past_events_sort($sort) {
      if(is_singular('event')) {
        global $wpdb;
        $new_sort = " GROUP BY p.ID ORDER BY m.meta_value+0 DESC";
        return $new_sort;
      }
      return $sort;
    }
    add_filter('get_previous_post_sort', 'get_adjacent_past_events_sort');
    add_filter('get_next_post_sort', 'get_adjacent_past_events_sort');