【Node.js】爬虫(puppeteer / cheerio)

puppeteer

安装：npm i puppeteer

就是自动控制浏览器进行点击，获取元素等操作。

import puppeteer  from "puppeteer";
// puppeteer 每个操作都是异步的

const input = process.argv[2]
// 默认无头模式，浏览器在后台操作
const browser = await puppeteer.launch({
    headless: false,
    executablePath: 'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe'  // 使用浏览器的路径(默认为Chrome)
})
// 开启页面
const page = await browser.newPage()
// 进行跳转
await page.setViewport({width: 1920, height: 1000})
await page.goto('https://www.zhihu.com/')
// 获取元素
await page.waitForSelector('.css-16vu25q')
const elements = await page.$$('.css-16vu25q .GlobalWriteV2-navTop .GlobalWriteV2-topTitle')
const collectContent = async () => {
    const titleList = []
    await page.waitForSelector('. css-0')
    const elements = await page.$$('. css-0 .css-1rs9lm3 .css-1o7uqnr')
    for await (const el of elements) {
        const prop = await el.getProperty('innerText')
        const text = await prop.jsonValue()
        titleList.push(text)
    }
    console.log(titleList)
}
for await (const el of elements) {
    const prop = await el.getProperty('innerText')
    const text = await prop.jsonValue()
    console.log(text)
    if (text === (input || '回答问题')) {
        await el.click()
        collectContent()
        break
    }
}

axios + cheerio

axios 用来在客户端或者服务端发送网络请求。

cheerio 用来对 html 标签元素进行一系列的操作，类似于 jQuery(实现了核心jQuery的子集)。

const axios = require('axios').default;  // default 是axios 在服务端的使用
const cheerio = require('cheerio');

/**
 * 获取 豆瓣读书首页的HTML
 * @returns HTML string
 */
async function getBooksHTML() {
  const result = await axios.get('https://book.douban.com/latest')
  return result.data;
}

/**
 * 获取豆瓣读书数据详情页链接的数组
 * @returns Array of book links
 */
async function getBookLinks() {
  const html = await getBooksHTML();
  const $ = cheerio.load(html);
  const achorElements = $('#content .grid-12-12 li a.cover');
  const links = achorElements.map((index, element) => {
    const href = element.attribs['href'];
    return href;
  }).get();
  return links;
}

/**
 * 根据书籍详情页的链接获取书籍的详细信息
 * @param {*} link 
 * @returns 书记详细信息的对象
 */
async function getBookDetail(link) {
  const result = await axios.get(link);
  const $ = cheerio.load(result.data);
  const name = $('h1').text().trim();
  const imgUrl = $('$mainpic .nbg img').attr('src');
  const spans = $('#info span.pl');
  const authorSpan = spans.filter((index, element) => {
    return $(element).text().includes('作者');
  })
  const author = authorSpan.naxt('a').text();
  const publishSpan = spans.filter((index, element) => {
    return $(element).text().includes('出版社');
  })
  const publishDate = publishSpan[0].nextSibling.nodeValue.trim();
  return {
    name,
    imgUrl,
    author,
    publishDate
  }
}

/**
 * 获取所有书籍的详细信息
 * @param {*} params 
 * @returns 
 */
async function fetchAll(params) {
  const links = await getBookLinks();
  const books = await Promise.all(links.map(link => getBookDetail(link)));
  return books;
}

async function saveToDB() {
  const books = await fetchAll();
  await bookModel.bulkCreate(books);
  console.log('数据保存成功');
}

fetchAll().then(books => console.log(books));