爬虫puppeteer-贝太美食网页列表demo（一）

在一个开始爬虫的时候，有些迷茫，至今也还只是实现了业务上的爬取。

用puppeteer去爬一些数据，真的可以事半功倍吧。

废话不多说，下面是爬取列表数据的代码。

// 'use strict';

const puppeteer = require('puppeteer');

const requestSys = require('request');

const fs = require('fs');

let dataList = []

let detailIdList = []

const urlArr = [

'https://site-api.beitaichufang.com/site/api/v1/index/days/meals/list',

'https://site-api.beitaichufang.com/site/api/v1/index/hot/video/list'

];

(async () => {

try {

const browser = await puppeteer.launch({

ignoreHTTPSErrors: true,

headless: false,

args: ['--no-sandbox', '--disable-setuid-sandbox'],

}).catch(() => browser.close);

const page = await browser.newPage();

await page.setRequestInterception(true);

var num = 0;

await page.on('request', request => {

// console.log(request.url(), '===============',request.resourceType())

//这就是请求的类型如果是图片类型的话执行abort拦截操作 否则continue继续请求别的

// 一日三餐

if (request.resourceType() === 'xhr' && (request.url() === urlArr[0] )){

getList(request.url(),'中式', 'zhongshi')

// 烘培

} else if(request.resourceType() === 'xhr' && (request.url() === urlArr[1])) {

getList(request.url(),'烘培', 'hongpei')

} else {

request.continue();

}

});

await page.goto('http://www.beitaichufang.com/');

await browser.close();

} catch (e) {

console.log(e);

}

})();

 

function getList(url, type, dataJson) {

requestSys({

url: url, // 请求的URL

method: 'POST', // 请求方法

headers: { // 指定请求头

'Accept-Language': 'zh-CN,zh;q=0.8', // 指定 Accept-Language

'Cookie': "gr_user_id=095f5bf6-43c3-4809-86bc-2770dbdf9de1; grwng_uid=bb9b7447-2a4f-4dae-ab1f-c9db626862f4; ab71762e6ffa91d3_gr_session_id=2bce554a-a40f-498e-ae53-17899c431d03; ab71762e6ffa91d3_gr_session_id_2bce554a-a40f-498e-ae53-17899c431d03=true"// 指定 Cookie

}

}, function (error, response, body) {

if (!error && response.statusCode == 200) {

dataList = []

let bodyList = JSON.parse(body).data.list

for(let i=0 ; i < bodyList.length ; i++){

if(dataJson === 'zhongshi') {

let daysMealsDirectoryList = bodyList[i].daysMealsDirectoryList

for(let j = 0 ; j<daysMealsDirectoryList.length; j++) {

let item = {}

item.converUrl = daysMealsDirectoryList[j].converUrl

item.desc = daysMealsDirectoryList[j].indexRecommendIntro

item.name = daysMealsDirectoryList[j].name

item.type = type

item.directorId = daysMealsDirectoryList[j].directoryNumber

dataList.push(item)

detailIdList.push(daysMealsDirectoryList[j].directoryNumber)

}

}

if(dataJson === 'hongpei') {

let item = {}

item.converUrl = bodyList[i].icon

item.desc = bodyList[i].intro

item.name = bodyList[i].name

item.type = type

item.directorId = bodyList[i].number

item.videoTime = bodyList[i].videoTime

dataList.push(item)

detailIdList.push(bodyList[i].number)

}

}

// 将数据写入

let writerStream = fs.createWriteStream(`./data/${dataJson}.json`);

let writerStream1 = fs.createWriteStream(`./data/detail/detailIdList.json`);

writerStream.write(JSON.stringify(dataList, undefined, 2), 'UTF8');

writerStream1.write(JSON.stringify(detailIdList, undefined, 2), 'UTF8');

writerStream.end();

}

});

}

上面的内容操作其实很简单，就是打开制定的网页，然后通过请求拦截，获取到与你需要的接口数据，然后自己根据需要重新组装数据。最后，由于当时没有做好连接数据库，所有暂时将数据写入到本地的一个文件夹下面。

 

有问题，欢迎指出！