说明
- 仅限于技术交流,不要干坏事
- 爬虫用的好,进去是迟早
- 文章我只是简单写的,并没有做优化或者封装,如果有需要,自己再优化
爬取表情包
const axios = require('axios');
const cheerio = require('cheerio');
let fs = require('fs');
const path = require('path');
const request = require('request');
let pathUrl = 'https://www.doutula.com/article/list/?page=1';
axios.get(pathUrl)
.then(res => {
let data = res.data;
let $ = cheerio.load(data);
$('#home .row .center-wrap a').each((i, ele) => {
let url = $(ele).attr('href');
if (url.indexOf('https') !== -1 && url.indexOf('detail') !== -1) {
axios.get(url).then(res => {
let content = res.data;
let $ = cheerio.load(content);
let img = $('.list-group-item .pic-content .artile_des table tr td a img');
let title = './img/' + $('.pic-title h1 a').text();
fs.mkdir(title, function(err) {
if (err) {
console.log(err);
} else {
img.each((i, e) => {
let url = $(e).attr('src');
request(url).pipe(fs.createWriteStream(title + '/' + path.parse(url).base))
})
}
})
})
}
})
})
.catch(err => console.log(err));
async function getData(url, dirpath) {
const res = await axios.get(url);
let $ = cheerio.load(res.data);
$('.pic-content .artile_des tr a img').each((i, ele) => {
let picSrc = $(ele).attr('src');
console.log(path.parse(picSrc));
let writeStream = fs.createWriteStream(dirpath + '/' + path.parse(picSrc).base);
axios.get(picSrc, { responseType: 'stream' }).then((res) => {
res.data.pipe(writeStream);
})
})
}
遇到的问题
一开始使用axios获取数据,但是一直打不开图片,图片文件大小也大了很多,怀疑是res.data的格式问题,改用request成功的