0

I'm working on a Hubspot Api and trying to fetch all blog posts through api, but the api limit is only 20 post each time and i need to get 1960 blog posts. So how i supposed to do that?

Is there any way to do this with ajax or java script or through a loop that increase limit every time

Here is my code:

<?php
    include 'wp-load.php';
    require_once(ABSPATH . "wp-admin" . '/includes/image.php');
    require_once(ABSPATH . "wp-admin" . '/includes/file.php');
    require_once(ABSPATH . "wp-admin" . '/includes/media.php');
    require_once(ABSPATH . "wp-admin" . '/includes/taxonomy.php');

    $apiKey = "9****966-***************";
    $getArgs = array('timeout' => 120);

    // Grab all blog posts . '&archived=false&offset=0&limit=300' total=1906
    $response = wp_remote_get('https://api.hubapi.com/content/api/v2/blog-posts?hapikey=' . $apiKey . '&archived=false&limit=1960&offset=0', $getArgs);
    $output = json_decode(wp_remote_retrieve_body($response), true);

    $results = $output['objects'];
    foreach ($results as $item) {
        $postStatus = 'publish';

        if ($item['blog_author']['email'] !== '') {

            $user = get_user_by('email', $item['blog_author']['email']);

            if ($user == false) {

                $user = wp_insert_user(array(
                    'user_login' => $item['blog_author']['email'],
                    'user_email' => $item['blog_author']['email'],
                    'first_name' => $item['blog_author']['full_name'],
                    'user_pass' => substr(str_shuffle('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'),
                        0, 10),
                ));
                $user = get_user_by('id', $user);
            }
        } else {
            $user = wp_get_current_user();
        }

        if ($item['state'] == 'DRAFT') {
            $postStatus = 'draft';
        } elseif ($item['state'] == 'SCHEDULED') {
            $postStatus = 'future';
        }
        $categories = $item['topic_ids'];
        $args = array(
            'posts_per_page' => 1,
            'post_type' => 'post',
            'meta_key' => 'hubspot_id',
            'meta_value' => $item['id'],
            'meta_compare' => '='
        );

        $exists = new WP_Query($args);

        if ($exists->post_count == 0) {
            $post_id = wp_insert_post(array(
                'post_title' => $item['name'],
                'post_content' => $item['post_body'],
                'post_status' => $postStatus,
                //'post_category' => $categories,
                'post_type' => 'post',
                'post_date' => date('Y-m-d H:i:s', $item['publish_date'] / 1000),
                'post_author' => $user->ID
            ));
        } else {
            while ($exists->have_posts()) {
                $exists->the_post();
                $post_id = wp_update_post(array(
                    'ID' => get_the_ID(),
                    'post_title' => $item['name'],
                    'post_content' => $item['post_body'],
                    'post_status' => $postStatus,
                    //'post_category' => $categories,
                    'post_type' => 'post',
                    'post_date' => date('Y-m-d H:i:s', $item['publish_date'] / 1000),
                    'post_author' => $user->ID
                ));
            }
            wp_reset_postdata();
        }
        if (!empty($post_id) && !is_null($post_id)) {
            $image_url = $item['featured_image'];
            if ($image_url !== '' && isset($image_url)) {
                $imageAltText = $item['featured_image_alt_text'];

                $temp_file = download_url($image_url);
                $file = array(
                    'name' => basename($image_url), // ex: wp-header-logo.png
                    'type' => wp_check_filetype($temp_file, null)['type'],
                    'tmp_name' => $temp_file,
                    'error' => 0,
                    'size' => filesize($temp_file),
                );

                $overrides = [
                    'test_form' => false,
                    'test_size' => true,
                ];

                $results = wp_handle_sideload($file, $overrides);
                if (empty($results['error'])) {
                    $filename = $results['file']; // Full path to the file
                    $local_url = $results['url'];  // URL to the file in the uploads dir
                    $type = $results['type']; // MIME type of the file

                    // Set attachment data
                    $attachment = [
                        'post_mime_type' => $type,
                        'post_title' => sanitize_file_name($imageAltText),
                        'post_content' => '',
                        'post_status' => 'inherit'
                    ];

                    // Create the attachment
                    $attach_id = wp_insert_attachment($attachment, $filename, $post_id);
                    if (has_post_thumbnail($post_id)) {
                        $oldthumbnail = get_post_thumbnail_id($post_id);
                        wp_delete_attachment($oldthumbnail, true);
                    }

                    // Define attachment metadata
                    $attach_data = wp_generate_attachment_metadata($attach_id, $filename);

                    // Assign metadata to attachment
                    wp_update_attachment_metadata($attach_id, $attach_data);

                    // And finally assign featured image to post
                    set_post_thumbnail($post_id, $attach_id);
                }
            }
            if (!add_post_meta($post_id, 'hubspot_id', $item['id'], true)) {
                update_post_meta($post_id, 'hubspot_id', $item['id']);
            }
            wp_reset_postdata();
        }

        $topicIds = $item['topic_ids'];
        $getArgs = array('timeout' => 120);
        $topics = [];
        foreach ($topicIds as $topicId) {

            $response = wp_remote_get('https://api.hubapi.com/blogs/v3/topics/' . $topicId . '?hapikey=9****966-***************', $getArgs);
            $topics[] = json_decode(wp_remote_retrieve_body($response), true);

            foreach ($topics as $topic) {
                $catarr = array(
                    'cat_name' => $topic['name'],
                    'category_description' => $topic['description'],
                    'category_nicename' => $topic['slug'],
                    'taxonomy' => 'category'
                );
                $category_id = wp_insert_category($catarr);
                wp_set_post_terms($post_id, $category_id, "category", true);
                $default_category = (int)get_option('default_category');
                if (in_category($default_category, $post_id)) {
                    // get list of all the post categories
                    $post_categories = get_the_category($post_id);

                    // count the total of the categories
                    $total_categories = count($post_categories);

                    // check if the post is in more than 1 category (the default one and more..)
                    if ($total_categories > 1) {
                        // remove the default category from the post
                        wp_remove_object_terms($post_id, $default_category, 'category');
                    }
                }
            }
        }
    }
?>
  • Assume you mean the Hubspot API limit is 20? Have you thought about using the posts_per_page option and the offset parameter? Pagination Codex and WPSE Question – Bysander Aug 21 '18 at 14:33
  • I am inserting posts from api outside of wordpress environment and able to add only 20 post each time as per Api limit, total post count in 1906 and if i add limit to infinit i.e. 99999 the api call will stops at 125 posts and the categories of the post at this limit not working means all post goes to uncategorized so i need to call only 20 post each time but how i suppose to part all posts in chunks so after every time only 20 posts will insert and then the offset will change automatically thats i cant figure out how to code it. – Dev65 Aug 22 '18 at 9:03
0

You'd probably want to wrap what you've done up there into a function and do something like.

for ( $counter = 0; $counter <= 1906; $counter += 20) {

    $responseURL = 'https://api.hubapi.com/content/api/v2/blog-posts?hapikey=' . $apiKey . '&archived=false&limit=20&offset=' . $counter;

    $response = wp_remote_get($responseURL, $getArgs);
    process_API_results($response);

    $restcounter++;

    if($restcounter == 6) { // works out to 120 posts then stops for 2 mins to prevent Hubspot stopping sync
        sleep(120);
        $restcounter = 0;
    }
}

If you echo out those URL requests you get

https://api.hubapi.com/content/api/v2/blog-posts?hapikey=&archived=false&limit=20&offset=0 https://api.hubapi.com/content/api/v2/blog-posts?hapikey=&archived=false&limit=20&offset=20 https://api.hubapi.com/content/api/v2/blog-posts?hapikey=&archived=false&limit=20&offset=40 https://api.hubapi.com/content/api/v2/blog-posts?hapikey=&archived=false&limit=20&offset=60 https://api.hubapi.com/content/api/v2/blog-posts?hapikey=&archived=false&limit=20&offset=80

... etc

  • Thanks for answering @Bysander I tried this but it stops at 125 posts i need to add interval or any thing that stop the execution after 20 posts and starts it automatically after some interval – Dev65 Aug 23 '18 at 6:24
  • See the update - you can use sleep() function to make the script wait - I've put 2 mins (120 seconds) but this may be overkill – Bysander Aug 23 '18 at 10:08

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.