一、代码
var https = require('https');
var cheerio = require('cheerio');
var mysql = require('mysql');
var table = "job";
var moment = require('moment');
var today = moment();
var year = today.format('YYYY年');
var yesterday = today.subtract(1, 'days').format('YYYY年MM月DD日');
var urlTool = require("url");
var qs = require('querystring');
var db_config = {
host: '127.0.0.1',
user: 'root',
password:'123456',
port:'3306',
database:'node'
};
var connection;
function handleDisconnect() {
connection = mysql.createConnection(db_config);
connection.connect(function(err) {
if(err) {
console.log('进行断线重连:' + new Date());
setTimeout(handleDisconnect, 2000); //2秒重连一次
return;
}
//console.log('连接成功');
});
connection.on('error', function(err) {
console.log('db error', err);
if(err.code === 'PROTOCOL_CONNECTION_LOST') {
handleDisconnect();
} else {
throw err;
}
});
}
function filterJobItem(html){
var $ = cheerio.load(html, {decodeEntities: false});
var jobItem = $('.job-list').find('ul').children('li');
var jobData = [];
var msgs = [];
var companyMsgs = [];
var titleAndSalarys = [];
var item, primaryInfo, companyInfo, id, msg, companyMsg, companyShortName, titleAndSalary, date;
var addJob = 'insert into job(titleId, title, salary, city, workYear, education, companyName, industry, financing, companySize,date) values(?,?,?,?,?,?,?,?,?,?,?)';
var i;
jobItem.each(function(item){
item = $(this);
primaryInfo = item.find('.job-primary').find('.info-primary');
companyInfo = item.find('.job-primary').find('.info-company');
id = primaryInfo.find('.name').find('a').attr('href').split('/')[2].split('.')[0];
titleAndSalary = primaryInfo.find('.name').find('a').text();
titleAndSalarys = titleAndSalary.split(' ');
msg = primaryInfo.find('p').html();
msgs = msg.split('<em class="vline"></em>');
companyShortName = companyInfo.find('.company-text').find('.name').find('a').text();
companyMsg = companyInfo.find('p').html();
companyMsgs = companyMsg.split('<em class="vline"></em>');
date = item.find('.job-time').text().substr(3);
if('昨天' === date)
date = yesterday;
else {
date = year + date;
}
handleDisconnect();
param = [id,titleAndSalarys[0],titleAndSalarys[1],msgs[0],msgs[1],msgs[2],companyShortName,companyMsgs[0],companyMsgs[1],companyMsgs[2],date];
connection.query(addJob, param, function(error, result){
if(error){
console.log(error.message);
}else{
console.log('insert id: ' + result.titleId);
}
});
})
return jobData;
}
var url;
function httpGet(url){
https.get(url,function(res){
var html = '';
res.on('data',function(data){
html += data;
})
res.on('end',function(){
//console.log(html);
var jobData = filterJobItem(html);
})
}).on('error', function(){
console.log('出错了!');
})
}
var pages = [];
function robot(){
pages.forEach(function(page){
url = url + qs.stringify({ query: 'Java', page: page, ka: 'page-1' },'&');
httpGet(url);
});
}
robot();
二、操作
1.1、引入依赖
var https = require('https');//取决于你要爬网址是http还是https
var cheerio = require('cheerio');//用法类似jquery
var mysql = require('mysql');
var moment = require('moment');//格式化时间
var urlTool = require("url");
var qs = require('querystring');//url和querystring是nodejs用来处理url的两大利器
1.2、依赖的简单使用
1.2.1 monent
var today = moment();
var year = today.format('YYYY年');//2017年
var yesterday = today.subtract(1, 'days').format('YYYY年MM月DD日');//昨天的日期,格式为:2017年11月24日
1.2.2 url和querystring
var getQuery = urlTool.parse(url).query;
var getData = qs.parse(getQuery);//{ query: 'Java', page: '1', ka: 'page-1' }
getData = qs.stringify({ query: 'Java', page: '4', ka: 'page-1' },'&');//query=Java&page=4&ka=page-1
1.2.3 mysql
//nodejs-mysql断线重连
var db_config = {
host: '127.0.0.1',
user: 'root',
password:'123456',
port:'3306',
database:'node'
};
var connection;
function handleDisconnect() {
connection = mysql.createConnection(db_config);
connection.connect(function(err) {
if(err) {
console.log('进行断线重连:' + new Date());
setTimeout(handleDisconnect, 2000); //2秒重连一次
return;
}
//console.log('连接成功');
});
connection.on('error', function(err) {
console.log('db error', err);
if(err.code === 'PROTOCOL_CONNECTION_LOST') {
handleDisconnect();
} else {
throw err;
}
});
}
1.2.4 cheerio(用法类似jquery)
item = $(this);
id = primaryInfo.find('.name').find('a').attr('href').split('/')[2].split('.')[0];
titleAndSalary = primaryInfo.find('.name').find('a').text();
三、爬虫的使用
将要爬取网页的页数以数组的形式传入pages,在nodejs 执行 node server.js即可。