Okay I found a solution after Slack conversation. It is possible to remove the fields from checkout by unsetting them in woocommerce_billing_fields
hook and then re-adding the values again to the order from users profile with woocommerce_checkout_posted_data
hook.
add_filter( 'woocommerce_checkout_posted_data', 'fill_order_billing_details' );
function fill_order_billing_details( $data ) {
$customer_id = get_current_user_id();
$data['billing_first_name'] = get_user_meta( $customer_id, 'billing_first_name', true );
$data['billing_last_name'] = get_user_meta( $customer_id, 'billing_last_name', true );
$data['billing_company'] = get_user_meta( $customer_id, 'billing_company', true );
$data['billing_country'] = get_user_meta( $customer_id, 'billing_country', true );
$data['billing_address_1'] = get_user_meta( $customer_id, 'billing_address_1', true );
$data['billing_address_2'] = get_user_meta( $customer_id, 'billing_address_2', true );
$data['billing_city'] = get_user_meta( $customer_id, 'billing_city', true );
$data['billing_state'] = get_user_meta( $customer_id, 'billing_state', true );
$data['billing_postcode'] = get_user_meta( $customer_id, 'billing_postcode', true );
$data['billing_phone'] = get_user_meta( $customer_id, 'billing_phone', true );
$data['billing_email'] = get_user_meta( $customer_id, 'billing_email', true );
return $data;
}