爬虫主要是爬去网络上网页抓取数据,然后分析提炼数据做我们想做的事。
通俗点来讲就是发送http请求抓取网页内容然后解析网页。或者直接发送http请求获取数据。
var http = require('http');
var iconv = require('../node_modules/iconv-lite');//用于编码防止网页乱码
class httpClient {
constructor(url,options,encoding) {
this.url = url;
this.encoding = encoding;
this.options = {
host: 'www.jokeji.cn',
port: '80',
path: '/jokehtml/ym/2017092623154972.htm'
};
Object.assign(this.options,options)
}
request(callback) {
let self = this;
var req = http.request(this.options,function(response){
// 不断更新数据
var body = '';
response.on('data', function(data) {
if(self.encoding){
body += iconv.decode(data, self.encoding);
}else{
body += data;
}
});
response.on('end', function() {
console.log(body)
// 数据接收完成
callback(body)
});
})
req.end();
}
get(callback){
let self = this;
var req = http.get(this.url,function(response){
// 不断更新数据
var body = '';
response.on('data', function(data) {
if(self.encoding){
body += iconv.decode(data, self.encoding);
}else{
body += data;
}
});
response.on('end', function() {
// 数据接收完成
callback(body)
});
})
}
}
module.exports = httpClient;
封装http请求方法,通过传入回调函数解析获取的网页内容。
比如我们可以像如下这样使用
const httpClient = require('../src/httpClient');
var cheerio = require('cheerio');//服务端的jquery
var options = {
host: 'astro.fashion.qq.com',
port: '80',
path: '/06newver/horoscope.shtml'
}
var url = "http://astro.fashion.qq.com/06newver/horoscope.shtml";
var self = this;
new httpClient(url, {}, 'GB2312').get(function(html) {
var $ = cheerio.load(html, { decodeEntities: false }); //采用cheerio模块解析html
$("#yunshi #fod1list p span a").each(function() {
var url = $(this).attr('href');
if (url.indexOf('dayastro') !== -1) {
self.getHtmlDetail(url);
}
})
})