使用async批量获取博客链接

主要使用了superagent  cheerio  async

目的获取所有的博客链接，需要的参数有 博客地址 以及 总页码

 

let superagent = require('superagent')
let cheerio = require('cheerio')
const async = require('async')

let baseBlogLink = 'https://fizzz.blog.csdn.net/'  // 博客地址
let articleSelector = '.article-list h4 a' // 文章选择器
let totalPage = 9 // 总页码


const setData = {
  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
  'Referrer': 'https://fizzz.blog.csdn.net/',
  'Content-Type': 'text/html; charset=UTF-8',
}

// Pretend this is some complicated async factory
var getOnePageBlogLink = function(pageIndex, callback) {
  pageIndex = pageIndex + 1
  superagent
    .get(`${baseBlogLink}/article/list/${pageIndex}`)
    .set(setData)
    .then(res => {
      let blogHrefArr = []
      let $ = cheerio.load(res.text)
      let allEle = $(articleSelector)
      let len = allEle.length
      if (len > 0) {
        console.log(`获取到${len}条博客记录`)
        for (let i = 0; i < len; i++) {
          let blogItem = {
            name: allEle.eq(i).text().replace(/\s+/g, ''),
            href: allEle.eq(i).attr('href'),
          }
          blogHrefArr.push(blogItem)
        }
      }
      callback(null, blogHrefArr)
    })
};

async.times(totalPage, function(n, next) {
  getOnePageBlogLink(n, function(err, blogs) {
    next(err, blogs);
  });
}, function(err, blogsArr) {
  const allBlogs = blogsArr.flat()
  console.log(allBlogs)
  console.log(allBlogs.length, 123)
});

 

执行效果图