I was searching for this for the last one week but found nothing. Hopefully got this query to display specific category posts only:

    <?php
        $years = $wpdb->get_col("SELECT DISTINCT YEAR(post_date) FROM $wpdb->posts INNER JOIN $wpdb->term_relationships ON ($wpdb->posts.ID = $wpdb->term_relationships.object_id) INNER JOIN $wpdb->term_taxonomy ON ($wpdb->term_relationships.term_taxonomy_id = $wpdb->term_taxonomy.term_taxonomy_id AND $wpdb->term_taxonomy.taxonomy='category') INNER JOIN $wpdb->terms ON ($wpdb->terms.term_id = $wpdb->term_taxonomy.term_id) WHERE $wpdb->terms.slug='blog' ORDER BY post_date DESC");
		foreach($years as $year) : 
		?>
    <li><a href="JavaScript:void()"><?php echo $year; ?></a>        
        <ul class="archive-sub-menu">
        <?  $months = $wpdb->get_col("SELECT DISTINCT MONTH(post_date)
        FROM $wpdb->posts WHERE post_status = 'publish' AND post_type = 'post'
        AND YEAR(post_date) = '".$year."' ORDER BY post_date DESC");
        foreach($months as $month) :
        ?>
    <li><a href="<?php echo get_month_link($year, $month); ?>">
                <?php echo date( 'F', mktime(0, 0, 0, $month) );?></a>
            </li>
        <?php endforeach;?>
        </ul>
        </li>
		<?php endforeach; ?>