csv文件流读取,可以应对大文件,数据截断发送,不会出现内存不足的情况
function readInAppEventReports(user,date,filePath, callback) {
var reports = [];
var google_facebook_reports=[];
var other_reports=[];
var item_other = {};
var item_fg={};
var dbFlag = true;
var getCount = 0;
var tCount = 0;
var otherCount=0;
var facebook_google_Count=0;
var other_reports_send_config=5000;
var google_facebook_reports_send_config=5000;
current_site={
id : filePath.split('=')[1].split('&')[0]
};
async.waterfall([
function (cb) {
readStream = fs.createReadStream(filePath);
console.log('---test', filePath);
readStream.pipe(csv())
.on('data', function (data) {
var media_source=data['Media Source'];
if (media_source==="googleadwords_int" || media_source==="Facebook Ads") {
item_fg=packItemForInAppEventReportFacebookGoogle(data);
if (item_fg.err) {
console.log('item_fg.err',item_fg.err);
return callback(null);
} else {
eventDataFilterFacebookGoogle(item_fg.value,google_facebook_reports);
}
} else {
item_other = packItemForInAppEventReport(data);
if (item_other.err) {
console.log('item_other.err', item_other.err);
return callback(null);
} else {
eventDataFilter(item_other.value, other_reports);
}
}
tCount++;
async.waterfall([
function (fn) {
if (other_reports.length >= other_reports_send_config && dbFlag) {
console.log('on data... send before other_reports.length=',other_reports.length);
dbFlag = false;
readStream.pause();
var temOther = other_reports.slice(0, other_reports_send_config);
Dbapi.sendMsg({
json: {
actionid: 8220,
noat: true,
events: temOther
}
}, function (err, result) {
console.log('other_reports Dbapi sendMsg', 8220, err, result);
getCount += temOther.length;
otherCount+=temOther.length;
console.log('on data other_reports tCount', tCount, 'getCount', getCount,'otherCount',otherCount);
other_reports.splice(0, temOther.length);
console.log('on data... after other_reports.length=',other_reports.length);
dbFlag = true;
readStream.resume();
fn(err);
}, 2);
} else {
fn(null);
}
},
function (fn) {
if (google_facebook_reports.length >= google_facebook_reports_send_config && dbFlag) {
console.log('on data ...send before google_facebook_reports.length=',google_facebook_reports.length);
dbFlag = false;
readStream.pause();
var temp_google_facebook_ = google_facebook_reports.slice(0, google_facebook_reports_send_config);
Dbapi.sendEvent({
json: {
actionid: 1001,
noat: true,
events: temp_google_facebook_
}
}, function (err, result) {
console.log('google_facebook_reports Dbapi sendMsg', 1001, err, result);
getCount += temp_google_facebook_.length;
facebook_google_Count+=temp_google_facebook_.length;
console.log('on data google_facebook_reports tCount', tCount, 'getCount', getCount,'facebook_google_Count',facebook_google_Count);
google_facebook_reports.splice(0, temp_google_facebook_.length);
console.log('on data... send after google_facebook_reports.length=',google_facebook_reports.length);
dbFlag = true;
readStream.resume();
fn(err);
}, 2);
} else {
fn(null);
}
}
],function (err) {
if (err) console.log('readInAppEventReports on(data) ',err);
if (tCount>=API_MAX) {
reports.push(data);
}
});
})
.on('end', function () {
var otherMsgSend=function (buff,cb) {
Dbapi.sendMsg({
json: {
actionid: 8220,
noat: true,
events: buff
}
}, function (err, result) {
console.log('other_reports Dbapi sendMsg', 8220, err, result);
getCount += buff.length;
otherCount+=buff.length;
console.log('other_reports *** tCount', tCount, 'getCount', getCount,'otherCount',otherCount);
cb(err);
}, 2);
};
var fgMsgSend=function (buff,cb) {
Dbapi.sendEvent({
json: {
actionid: 1001,
noat: true,
events: buff
}
}, function (err, result) {
console.log('google_facebook_reports Dbapi sendMsg', 1001, err, result);
getCount += buff.length;
facebook_google_Count+=buff.length;
console.log('google_facebook_reports *** tCount', tCount, 'getCount', getCount,'facebook_google_Count',facebook_google_Count);
cb(err);
}, 2);
};
async.waterfall([
function (fn) {
console.log('on end read other_reports');
console.log('on end send before other_reports.length =',other_reports.length );
if (other_reports.length !== 0) {
var flag=true;
async.whilst(
function () {
return flag;
},
function (fn_cb) {
console.log('on end send..other_reports.length =',other_reports.length );
if (other_reports.length > other_reports_send_config){
flag=true;
var buff=other_reports.splice(0,other_reports_send_config);
otherMsgSend(buff,function (err) {
fn_cb(err);
});
} else {
flag=false;
return fn_cb(null)
}
},
function (err) {
if (err) console.log(err);
otherMsgSend(other_reports,function (error) {
fn(error);
});
});
} else {
fn(null)
}
},
function (fn) {
console.log('on end read google_facebook_reports');
console.log('on end send before google_facebook_reports.length=',google_facebook_reports.length);
if (google_facebook_reports.length !==0) {
var flag=true;
async.whilst(
function () {
return flag;
},
function (fn_cb) {
console.log('on end send...google_facebook_reports.length=',google_facebook_reports.length);
if (google_facebook_reports.length > google_facebook_reports_send_config) {
flag=true;
var buff=google_facebook_reports.splice(0,google_facebook_reports_send_config);
fgMsgSend(buff,function (err) {
fn_cb(err);
});
} else {
flag=false;
return fn_cb(null);
}
},
function (err) {
if (err) console.log(err);
fgMsgSend(google_facebook_reports,function (error) {
fn(error);
});
});
} else {
fn(null);
}
}
],function (err) {
if (err) console.log('readInAppEventReports on(end) ',err);
cb(null);
});
});
},
function (cb) {
cb(null);
}
], function (err) {
if (err) console.log(err);
async.waterfall([
function (fn) {
console.log('tCount=',tCount,'API_MAX=',API_MAX);
if (tCount >= API_MAX - 20000) {
console.log('readInAppEventReports API_MAX - 20000 Monitor ...');
var apiErr = Monitor.errFactory();
apiErr.err = '数据不完整';
apiErr.funct = readInstallsReport.toString();
apiErr.param = filePath;
apiErr.level = 10;
Monitor.send(apiErr);
}
fn(null);
},
function (fn) {
console.log('tCount=',tCount,'API_MAX=',API_MAX);
if (tCount >= API_MAX) {
console.log('readInAppEventReports data >',API_MAX/1000,'K');
var detail_date=[];
detail_date=reports[0]['Event Time'].substr(10,6).split(':');
detail_date[0]=detail_date[0].replace(" ","");
if (parseInt(detail_date[1])<59) {
detail_date[1]=parseInt(detail_date[1])+1;
}
if (parseInt(detail_date[1])===59) {
detail_date[0]=parseInt(detail_date[0])+1;
detail_date[1]='00';
}
console.log('detail_date:',detail_date);
var param={
detail_date:detail_date,
website:current_site
};
setTimeout(function () {
inAppEventReport(user, date, param, function (err) {
console.log('inAppEventReport end err', err);
Monitor.send('read event report');
fn(err);
});
},3000);
} else if (tCount===6 || other_reports.length===0 || google_facebook_reports.length===0) {
console.log('当前时间',new Date().toLocaleDateString(),'待获取应用日期 date=',date,' website= ',current_site.id,'应用超过请求次数');
fn(null);
} else {
console.log('当前时间',new Date().toLocaleDateString(),'待获取应用日期 date=',date,' website= ',current_site.id,'该天没有数据');
fn(null);
}
}
],function (err) {
callback(err);
});
});
}
集合遍历,eachLimit里面的方法会并行
async.eachLimit(users[user].websites, users[user].websites.length, function (website, cb) {
console.log('inAppEventReports user=',user,'date=',date,'website=',website);
loadInAppEventReports(user, website, date, function (err, path) {
if (path) filesPath = filesPath.concat(path);
console.log('loadInAppEventReports end', website.id, 'err', err, new Date().toString());
cb(null);
})
}, function (err) {
cba(err, filesPath);
});
可以将limit设置为1 使其串行
async.eachLimit(dateList.slice(0,6), 1, function (dateItem, callback) {
console.log('waterfall-start', dateItem);
async.waterfall([
function (cb) {
mainFunction(dateItem, dateItem, function (err) {
cb(null);
});
}
], function (err) {
callback(null);
});
}, function (err) {
console.error('history end err', err);
});