nodejs 爬取表情包

说明

• 仅限于技术交流，不要干坏事
• 爬虫用的好，进去是迟早
• 文章我只是简单写的，并没有做优化或者封装，如果有需要，自己再优化

爬取表情包

const axios = require('axios');
const cheerio = require('cheerio');
let fs = require('fs');
const path = require('path');
const request = require('request');

let pathUrl = 'https://www.doutula.com/article/list/?page=1';

axios.get(pathUrl)
    .then(res => {
        let data = res.data;
        let $ = cheerio.load(data);
        $('#home .row .center-wrap a').each((i, ele) => {
            let url = $(ele).attr('href');
            if (url.indexOf('https') !== -1 && url.indexOf('detail') !== -1) {

                axios.get(url).then(res => {
                    let content = res.data;
                    let $ = cheerio.load(content);
                    let img = $('.list-group-item .pic-content .artile_des table tr td a img');
                    let title = './img/' + $('.pic-title h1 a').text();
                    fs.mkdir(title, function(err) {
                        if (err) {
                            console.log(err);
                        } else {
                            img.each((i, e) => {

                                let url = $(e).attr('src');
                                request(url).pipe(fs.createWriteStream(title + '/' + path.parse(url).base))
                            })
                        }
                    })

                })
            }
        })
    })
    .catch(err => console.log(err));


async function getData(url, dirpath) {
    const res = await axios.get(url);
    let $ = cheerio.load(res.data);
    $('.pic-content .artile_des tr a img').each((i, ele) => {
        let picSrc = $(ele).attr('src');
        console.log(path.parse(picSrc));
        let writeStream = fs.createWriteStream(dirpath + '/' + path.parse(picSrc).base);
        axios.get(picSrc, { responseType: 'stream' }).then((res) => {
            res.data.pipe(writeStream);
        })
    })
}

遇到的问题

一开始使用axios获取数据，但是一直打不开图片，图片文件大小也大了很多，怀疑是res.data的格式问题，改用request成功的