nodejs的puppeteer爬取百度指定数量的url,记录url也可以改成下载那些图片
const puppeteer=require('...下载puppeteer的绝对路劲或着放到本地主js在同一个目录./puppeteer');
const fs=require('fs');
const https=require('https');
const path=require('path');
const readline=require('readline');
const fs_e=require('同上./fs-extra');
(async()=>{
console.log("it's loading...");
const browser=await puppeteer.launch({handless:false,devtools:false});
const page=await browser.newPage();
await page.setViewport({
width:1098,
height:1000
})
await page.goto('https://image.baidu.com/');
console.log('go to site');
const read=readline.createInterface({
input:process.stdin,
output:process.stdout
});
read.question("what do you search >>>",async(searchstr)=>{
await page.waitForSelector('.input-container_94h2R input');
await page.focus('.input-container_94h2R input');
await page.keyboard.sendCharacter(searchstr);
await page.click('.submit-btn_ZmEXZ input');
read.question("how much img do you fetch >>>",async(num)=>{
// const intervalId=await setTimeout(async ()=>{await geturl(num)},5000);
await geturl(num);
read.close();
});
});
await fs_e.writeFile('百度url.csv','num,url\n');
let count=0;
var intervalId;
let record=[0];
let temp=0;
async function scroll(n){
try{
await page.evaluate(()=>{
window.scrollTo(0,document.body.scrollHeight);
})
await geturl(n);
}catch(err){}
}
async function check(rec,n){
record.push(rec);
if(record.length>2){
record.shift();
}
if(record.length>1&&record[0]!=record[1]){
}
if(record[0]==record[1]){
record.shift();
intervalId=await setTimeout(async ()=>{await scroll(n)},1000);
}
}
async function geturl(n){
try{
await page.waitForSelector('.imgpage');
let boxs=await page.$$('.imgpage');
await check(boxs.length,n);
for(let i=record[0];i<record[1];i++){
let imgs=await boxs[i].$$eval('.imgpage img',img=>{return img.map(img=>img.src)});//map是用来迭代的,$$eval函数的作用就是获取imgpage里面所有的img标签
for(const imgurl of imgs){
if(!imgurl.includes('base64')&&imgurl!==''){
count++;
if(count<=n){
console.log(imgurl);
await fs_e.appendFile('百度url.csv',`"${count}","${imgurl}"\n`);
}else{
if(typeof intervalId===undefined){
clearTimeout(intervalId);
}
browser.close();
}
}
}
}
intervalId=await setTimeout(async ()=>{await scroll(n)},1000);
}catch(err){}
}
})()