puppeteer
安装:npm i puppeteer
就是自动控制浏览器进行点击,获取元素等操作。
import puppeteer from "puppeteer";
// puppeteer 每个操作都是异步的
const input = process.argv[2]
// 默认无头模式,浏览器在后台操作
const browser = await puppeteer.launch({
headless: false,
executablePath: 'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe' // 使用浏览器的路径(默认为Chrome)
})
// 开启页面
const page = await browser.newPage()
// 进行跳转
await page.setViewport({width: 1920, height: 1000})
await page.goto('https://www.zhihu.com/')
// 获取元素
await page.waitForSelector('.css-16vu25q')
const elements = await page.$$('.css-16vu25q .GlobalWriteV2-navTop .GlobalWriteV2-topTitle')
const collectContent = async () => {
const titleList = []
await page.waitForSelector('. css-0')
const elements = await page.$$('. css-0 .css-1rs9lm3 .css-1o7uqnr')
for await (const el of elements) {
const prop = await el.getProperty('innerText')
const text = await prop.jsonValue()
titleList.push(text)
}
console.log(titleList)
}
for await (const el of elements) {
const prop = await el.getProperty('innerText')
const text = await prop.jsonValue()
console.log(text)
if (text === (input || '回答问题')) {
await el.click()
collectContent()
break
}
}
axios + cheerio
axios 用来在客户端或者服务端发送网络请求。
cheerio 用来对 html 标签元素进行一系列的操作,类似于 jQuery(实现了核心jQuery的子集)。
const axios = require('axios').default; // default 是axios 在服务端的使用
const cheerio = require('cheerio');
/**
* 获取 豆瓣读书首页的HTML
* @returns HTML string
*/
async function getBooksHTML() {
const result = await axios.get('https://book.douban.com/latest')
return result.data;
}
/**
* 获取豆瓣读书数据详情页链接的数组
* @returns Array of book links
*/
async function getBookLinks() {
const html = await getBooksHTML();
const $ = cheerio.load(html);
const achorElements = $('#content .grid-12-12 li a.cover');
const links = achorElements.map((index, element) => {
const href = element.attribs['href'];
return href;
}).get();
return links;
}
/**
* 根据书籍详情页的链接获取书籍的详细信息
* @param {*} link
* @returns 书记详细信息的对象
*/
async function getBookDetail(link) {
const result = await axios.get(link);
const $ = cheerio.load(result.data);
const name = $('h1').text().trim();
const imgUrl = $('$mainpic .nbg img').attr('src');
const spans = $('#info span.pl');
const authorSpan = spans.filter((index, element) => {
return $(element).text().includes('作者');
})
const author = authorSpan.naxt('a').text();
const publishSpan = spans.filter((index, element) => {
return $(element).text().includes('出版社');
})
const publishDate = publishSpan[0].nextSibling.nodeValue.trim();
return {
name,
imgUrl,
author,
publishDate
}
}
/**
* 获取所有书籍的详细信息
* @param {*} params
* @returns
*/
async function fetchAll(params) {
const links = await getBookLinks();
const books = await Promise.all(links.map(link => getBookDetail(link)));
return books;
}
async function saveToDB() {
const books = await fetchAll();
await bookModel.bulkCreate(books);
console.log('数据保存成功');
}
fetchAll().then(books => console.log(books));