在一个开始爬虫的时候,有些迷茫,至今也还只是实现了业务上的爬取。
用puppeteer去爬一些数据,真的可以事半功倍吧。
废话不多说,下面是爬取列表数据的代码。
// 'use strict';
const puppeteer = require('puppeteer');
const requestSys = require('request');
const fs = require('fs');
let dataList = []
let detailIdList = []
const urlArr = [
'https://site-api.beitaichufang.com/site/api/v1/index/days/meals/list',
'https://site-api.beitaichufang.com/site/api/v1/index/hot/video/list'
];
(async () => {
try {
const browser = await puppeteer.launch({
ignoreHTTPSErrors: true,
headless: false,
args: ['--no-sandbox', '--disable-setuid-sandbox'],
}).catch(() => browser.close);
const page = await browser.newPage();
await page.setRequestInterception(true);
var num = 0;
await page.on('request', request => {
// console.log(request.url(), '===============',request.resourceType())
//这就是请求的类型如果是图片类型的话执行abort拦截操作 否则continue继续请求别的
// 一日三餐
if (request.resourceType() === 'xhr' && (request.url() === urlArr[0] )){
getList(request.url(),'中式', 'zhongshi')
// 烘培
} else if(request.resourceType() === 'xhr' && (request.url() === urlArr[1])) {
getList(request.url(),'烘培', 'hongpei')
} else {
request.continue();
}
});
await page.goto('http://www.beitaichufang.com/');
await browser.close();
} catch (e) {
console.log(e);
}
})();
function getList(url, type, dataJson) {
requestSys({
url: url, // 请求的URL
method: 'POST', // 请求方法
headers: { // 指定请求头
'Accept-Language': 'zh-CN,zh;q=0.8', // 指定 Accept-Language
'Cookie': "gr_user_id=095f5bf6-43c3-4809-86bc-2770dbdf9de1; grwng_uid=bb9b7447-2a4f-4dae-ab1f-c9db626862f4; ab71762e6ffa91d3_gr_session_id=2bce554a-a40f-498e-ae53-17899c431d03; ab71762e6ffa91d3_gr_session_id_2bce554a-a40f-498e-ae53-17899c431d03=true"// 指定 Cookie
}
}, function (error, response, body) {
if (!error && response.statusCode == 200) {
dataList = []
let bodyList = JSON.parse(body).data.list
for(let i=0 ; i < bodyList.length ; i++){
if(dataJson === 'zhongshi') {
let daysMealsDirectoryList = bodyList[i].daysMealsDirectoryList
for(let j = 0 ; j<daysMealsDirectoryList.length; j++) {
let item = {}
item.converUrl = daysMealsDirectoryList[j].converUrl
item.desc = daysMealsDirectoryList[j].indexRecommendIntro
item.name = daysMealsDirectoryList[j].name
item.type = type
item.directorId = daysMealsDirectoryList[j].directoryNumber
dataList.push(item)
detailIdList.push(daysMealsDirectoryList[j].directoryNumber)
}
}
if(dataJson === 'hongpei') {
let item = {}
item.converUrl = bodyList[i].icon
item.desc = bodyList[i].intro
item.name = bodyList[i].name
item.type = type
item.directorId = bodyList[i].number
item.videoTime = bodyList[i].videoTime
dataList.push(item)
detailIdList.push(bodyList[i].number)
}
}
// 将数据写入
let writerStream = fs.createWriteStream(`./data/${dataJson}.json`);
let writerStream1 = fs.createWriteStream(`./data/detail/detailIdList.json`);
writerStream.write(JSON.stringify(dataList, undefined, 2), 'UTF8');
writerStream1.write(JSON.stringify(detailIdList, undefined, 2), 'UTF8');
writerStream.end();
}
});
}
上面的内容操作其实很简单,就是打开制定的网页,然后通过请求拦截,获取到与你需要的接口数据,然后自己根据需要重新组装数据。最后,由于当时没有做好连接数据库,所有暂时将数据写入到本地的一个文件夹下面。
有问题,欢迎指出!