node多进程跑phantomJs

一共有2个文件

1 lion.js

它主要是访问一个页面,来fetch相关的信息

2 manager.js

它主要来多开phantomJs

LION ->

var timeDog = require("./userLib/timeDog");
var getVideoDetail = require("./userLib/getVideoDetail");
var transData = require("./userLib/transData");
var fs = require("fs");
var args = require("system").args;

var dog = new timeDog();

var freshVideos;

var videoType = args[1],
	userName = args[2],
	userCode = args[3],
	timePoint = args[4];

// console.log(videoType, userName, userCode, timePoint);

var script = function() {};
var url = "http://i.youku.com/u/" + userCode + "/videos";
var page = require('webpage').create();
page.settings.loadImages = false;
page.settings.resourceTimeout=2000;
// 下面的写法是错误的
// page.settings = {
// 	// loadImages: false,
// 	 // resourceTimeout: 10*1000
// 	 resourceTimeout: 2000
// };
page.onResourceTimeout = function(request) {
	console.log('TIMEOUT');
	phantom.exit();
};
page.open(url, function(status) {

	if (status !== "success") {
		console.log("failed!!");
		return;
	}
	// console.log("open success!");
	page.injectJs("./includes/jquery.js");
	var list = page.evaluate(getVideoDetail);
	list = list.map(function(n, i) {
		return transData(n);
	}).filter(function(n, i) {
		return n.pub === timePoint;
	});

	page.close();
	fs.write([videoType, userName + ".json"].join("\\"), JSON.stringify(list, undefined, 4));
	console.log("COMPLETE");
	phantom.exit();
});

MANAGER ->

var async = require("async");
var child = require("child_process");
var fs = require("fs");
var timeDog = require("./userLib/timeDog");
require("./userLib/dateExt");

var dog = new timeDog("MANAGER");

var videoType = process.argv[2] || "";
var timespan = process.argv[3] || 0;

// var makerList = getMakers(videoType).slice(0, 1);
var makerList = getMakers(videoType);
// console.log(JSON.stringify(makerList, undefined, 4));
var timePoint = Date.today().addDays(-timespan).toFormatString();
var limit = 5;

// test();
// return;

var bagArr = [];

async.eachLimit(makerList, limit, function(item, next) {
	bagArr.push(item.userName);
	falcon(item.videoType, item.userName, item.userCode, timePoint, function(output) {
		dog.bark([item.userName, "RECIEVE:", output].join(" "));
		next(null);
		bagArr.forEach(function(n, i) {
			if (n == item.userName) {
				bagArr[i] = null;
			}
		});
		dog.bark(bagArr.filter(function(n, i) {
			return n != null;
		}).join("---"));
	});
}, function(err) {
	if (err) {
		dog.bark(["ERR:", err].join(" "));
	} else {

		dog.bark("Mission Complete!!")
	}
});

function getMakers(videoType) {
	var rst = [];
	var names = videoType ? [videoType] : fs.readdirSync("makers").map(function(fileName, i) {
		return fileName.replace("Maker.json", "");
	});
	names.forEach(function(videoType, i) {
		var path = ["makers", videoType + "maker.json", ].join("\\");
		var content = JSON.parse(fs.readFileSync(path, "utf-8")).map(function(user, i) {
			return {
				videoType: videoType,
				userName: user.name,
				userCode: user.code
			};
		});
		rst = rst.concat(content);
	});
	return rst;
}

function falcon(videoType, userName, userCode, timePoint, onComplete) {
	// var cmd = ["casperjs fetch.js", "--videoType=" + videoType, "--userName=" + userName, "--userCode=" + userCode, "--timePoint=" + timePoint].join(" ");
	var cmd = ["phantomjs lion.js", videoType, userName, userCode, timePoint].join(" ");
	dog.bark(cmd);
	child.exec(cmd, function(stdin, stdout, stderr) {
		console.log(stdout);
		if (stdout.indexOf("COMPLETE") >= 0) {
			// console.log(stdout);
			onComplete && onComplete(stdout);
		}
		if (stdout.indexOf("TIMEOUT") >= 0) {
			// console.log(stdout);
			onComplete && onComplete(stdout);
		}
	});
}
// test();

function test() {
	var videoType = "dota",
		userName = "820",
		userCode = "UMzc3MzI3Mzc2",
		timePoint = Date.today().toFormatString();
	falcon(videoType, userName, userCode, timePoint);
}