一共有2个文件
1 lion.js
它主要是访问一个页面,来fetch相关的信息
2 manager.js
它主要来多开phantomJs
LION ->
var timeDog = require("./userLib/timeDog");
var getVideoDetail = require("./userLib/getVideoDetail");
var transData = require("./userLib/transData");
var fs = require("fs");
var args = require("system").args;
var dog = new timeDog();
var freshVideos;
var videoType = args[1],
userName = args[2],
userCode = args[3],
timePoint = args[4];
// console.log(videoType, userName, userCode, timePoint);
var script = function() {};
var url = "http://i.youku.com/u/" + userCode + "/videos";
var page = require('webpage').create();
page.settings.loadImages = false;
page.settings.resourceTimeout=2000;
// 下面的写法是错误的
// page.settings = {
// // loadImages: false,
// // resourceTimeout: 10*1000
// resourceTimeout: 2000
// };
page.onResourceTimeout = function(request) {
console.log('TIMEOUT');
phantom.exit();
};
page.open(url, function(status) {
if (status !== "success") {
console.log("failed!!");
return;
}
// console.log("open success!");
page.injectJs("./includes/jquery.js");
var list = page.evaluate(getVideoDetail);
list = list.map(function(n, i) {
return transData(n);
}).filter(function(n, i) {
return n.pub === timePoint;
});
page.close();
fs.write([videoType, userName + ".json"].join("\\"), JSON.stringify(list, undefined, 4));
console.log("COMPLETE");
phantom.exit();
});
MANAGER ->
var async = require("async");
var child = require("child_process");
var fs = require("fs");
var timeDog = require("./userLib/timeDog");
require("./userLib/dateExt");
var dog = new timeDog("MANAGER");
var videoType = process.argv[2] || "";
var timespan = process.argv[3] || 0;
// var makerList = getMakers(videoType).slice(0, 1);
var makerList = getMakers(videoType);
// console.log(JSON.stringify(makerList, undefined, 4));
var timePoint = Date.today().addDays(-timespan).toFormatString();
var limit = 5;
// test();
// return;
var bagArr = [];
async.eachLimit(makerList, limit, function(item, next) {
bagArr.push(item.userName);
falcon(item.videoType, item.userName, item.userCode, timePoint, function(output) {
dog.bark([item.userName, "RECIEVE:", output].join(" "));
next(null);
bagArr.forEach(function(n, i) {
if (n == item.userName) {
bagArr[i] = null;
}
});
dog.bark(bagArr.filter(function(n, i) {
return n != null;
}).join("---"));
});
}, function(err) {
if (err) {
dog.bark(["ERR:", err].join(" "));
} else {
dog.bark("Mission Complete!!")
}
});
function getMakers(videoType) {
var rst = [];
var names = videoType ? [videoType] : fs.readdirSync("makers").map(function(fileName, i) {
return fileName.replace("Maker.json", "");
});
names.forEach(function(videoType, i) {
var path = ["makers", videoType + "maker.json", ].join("\\");
var content = JSON.parse(fs.readFileSync(path, "utf-8")).map(function(user, i) {
return {
videoType: videoType,
userName: user.name,
userCode: user.code
};
});
rst = rst.concat(content);
});
return rst;
}
function falcon(videoType, userName, userCode, timePoint, onComplete) {
// var cmd = ["casperjs fetch.js", "--videoType=" + videoType, "--userName=" + userName, "--userCode=" + userCode, "--timePoint=" + timePoint].join(" ");
var cmd = ["phantomjs lion.js", videoType, userName, userCode, timePoint].join(" ");
dog.bark(cmd);
child.exec(cmd, function(stdin, stdout, stderr) {
console.log(stdout);
if (stdout.indexOf("COMPLETE") >= 0) {
// console.log(stdout);
onComplete && onComplete(stdout);
}
if (stdout.indexOf("TIMEOUT") >= 0) {
// console.log(stdout);
onComplete && onComplete(stdout);
}
});
}
// test();
function test() {
var videoType = "dota",
userName = "820",
userCode = "UMzc3MzI3Mzc2",
timePoint = Date.today().toFormatString();
falcon(videoType, userName, userCode, timePoint);
}