var http = require('http');
var fs = require('fs');
var request = require('request');
var CryptoJS = require('crypto-js');
function decode(code) {
code = code.replace(/^eval/, '');
return eval(code);
}
function encode(code){
code = code.replace(/[\r\n]+/g, '');
code = code.replace(/'/g, "\\'");
var tmp = code.match(/\b(\w+)\b/g);
tmp.sort();
var dict = [];
var i, t = '';
for(var i=0; i<tmp.length; i++) {
if(tmp[i] != t) dict.push(t = tmp[i]);
}
var len = dict.length;
var ch;
for(i=0; i<len; i++) {
ch = num(i);
code = code.replace(new RegExp('\\b'+dict[i]+'\\b','g'), ch);
if(ch == dict[i]) dict[i] = '';
}
return "eval(function(p,a,c,k,e,d){e=function(c){return(c<a?'':e(parseInt(c/a)))+((c=c%a)>35?String.fromCharCode(c+29):c.toString(36))};if(!''.replace(/^/,String)){while(c--)d[e(c)]=k[c]||e(c);k=[function(e){return d[e]}];e=function(){return'\\\\w+'};c=1};while(c--)if(k[c])p=p.replace(new RegExp('\\\\b'+e(c)+'\\\\b','g'),k[c]);return p}(" + "'"+code+"',"+a+","+len+",'"+ dict.join('|')+"'.split('|'),0,{}))";
}
function downloadPic(url,filePath,fn){
var r = request(url).pipe(fs.createWriteStream(filePath));
r.on('close', function(){
fn();
}).on('error',function(){
log('download pic error!');
if(nextChapterPath && chapterNow < chapterNum){
downloadChapter(nextChapterPath);
}else{
chapterNow++;
}
});
}
function downloadFiles(arr,baseDir){
var length = arr.length;
var i = 0;
function loadNext(index){
downloadPic(encodeURI(picDomain + jsonData.path + arr[i]), baseDir + '/' + index + '.jpg',function(){
i++;
if(i < length){
console.log('Pictrure ' + (index + 1 + '/' + jsonData.len) + ' done!');
loadNext(i);
}else{
console.log('Chapter ' + jsonData.cname + ' All done!');
chapterNum++;
if(chapterNum < maxChapterNum){
downloadChapter(chapterNum + '.html');
}
}
});
}
loadNext(i);
}
var baseDir = 'F:/node_test/Pictures/';
var picDomain = 'http://somedomain.pic.com';
var dataReg = new RegExp("decryptDES\\(\\'([\\w+/=]+)\\'\\)");
var jsonData = null;
var chapterNum = 154731;
var maxChapterNum = 154743;
function log(data){
fs.appendFile(baseDir + 'log.txt',data + '\r\n','utf8',function(err){
if(err)
{
console.log(err);
}
});
}
function decryptDES(t) {
var a = CryptoJS.DES.decrypt({
ciphertext: CryptoJS.enc.Base64.parse(t.substring(8))
}, CryptoJS.enc.Utf8.parse(t.substring(0, 8)), {
mode: CryptoJS.mode.ECB,
padding: CryptoJS.pad.Pkcs7
});
return a.toString(CryptoJS.enc.Utf8);
}
function downloadChapter(path){
log('//------------------------------------------------------------------//');
var page = 'http://somepage.url.com/' + path;
log('正在读取页面:' + page);
var html = '';
http.get(page , function(res) {
res.setEncoding('utf-8');
res.on('data', function(data) {
// collect the data chunks to the variable named "html"
html += data;
}).on('end', function() {
var dir = baseDir;
if(dataReg.test(html)){
var str = decryptDES(RegExp.$1);
str = str.replace(/^eval/,'');
str = eval(str);
log('页面数据:\r\n' + str);
str = str.replace('var cInfo=','return ');
//字符串转对象
jsonData = (new Function(str))();
dir += jsonData["cname"];
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
log('文件存放目录:' + dir);
}else{
console.log(dir + ' 已存在!');
return;
}
downloadFiles(jsonData["files"],dir);
}else{
return;
}
});
});
}
downloadChapter( chapterNum + '.html');
爬取网站的图片
最新推荐文章于 2023-08-21 08:00:00 发布