nodejs爬取百度图片url放进csv文件里面

        nodejs的puppeteer爬取百度指定数量的url,记录url也可以改成下载那些图片

        

const puppeteer=require('...下载puppeteer的绝对路劲或着放到本地主js在同一个目录./puppeteer');
const fs=require('fs');
const https=require('https');
const path=require('path');
const readline=require('readline');
const fs_e=require('同上./fs-extra');

(async()=>{

    console.log("it's loading...");

    const browser=await puppeteer.launch({handless:false,devtools:false});
    const page=await browser.newPage();
    await page.setViewport({
        width:1098,
        height:1000
    })
    await page.goto('https://image.baidu.com/');
    console.log('go to site');


    const read=readline.createInterface({
        input:process.stdin,
        output:process.stdout
    });

    read.question("what do you search >>>",async(searchstr)=>{
        await page.waitForSelector('.input-container_94h2R input');
        await page.focus('.input-container_94h2R input');
        await page.keyboard.sendCharacter(searchstr);
        await page.click('.submit-btn_ZmEXZ input');

        read.question("how much img do you fetch >>>",async(num)=>{
            // const intervalId=await setTimeout(async ()=>{await geturl(num)},5000);
            await geturl(num);
            read.close();
        });
    });
    await fs_e.writeFile('百度url.csv','num,url\n');


    let count=0;
    var intervalId;
    let record=[0];
    let temp=0;
    async function scroll(n){
        try{
            await page.evaluate(()=>{
                window.scrollTo(0,document.body.scrollHeight);
            })

            await geturl(n);
            
        }catch(err){}
    }



    async function check(rec,n){
        record.push(rec);
        if(record.length>2){
            record.shift();
        }
        if(record.length>1&&record[0]!=record[1]){
        }
        if(record[0]==record[1]){
            record.shift();
            intervalId=await setTimeout(async ()=>{await scroll(n)},1000);
        }


    }

    async function geturl(n){
        try{
            await page.waitForSelector('.imgpage');
            let boxs=await page.$$('.imgpage');


            await check(boxs.length,n);



            for(let i=record[0];i<record[1];i++){
                let imgs=await boxs[i].$$eval('.imgpage img',img=>{return img.map(img=>img.src)});//map是用来迭代的，$$eval函数的作用就是获取imgpage里面所有的img标签
                for(const imgurl of imgs){
                    if(!imgurl.includes('base64')&&imgurl!==''){
                        count++;
                        if(count<=n){
                            console.log(imgurl);
                            await fs_e.appendFile('百度url.csv',`"${count}","${imgurl}"\n`);
                        }else{
                            if(typeof intervalId===undefined){
                                clearTimeout(intervalId);
                            }
                            browser.close();
                        }
                    }
                }
            }
            intervalId=await setTimeout(async ()=>{await scroll(n)},1000);
        }catch(err){}

    }
    
})()