node csv文件流读取

csv文件流读取，可以应对大文件,数据截断发送，不会出现内存不足的情况
function readInAppEventReports(user,date,filePath, callback) {
    var reports = [];
    var google_facebook_reports=[];
    var other_reports=[];
    var item_other = {};
    var item_fg={};
    var dbFlag = true;
    var getCount = 0;
    var tCount = 0;     //api获取数据条数;
    var otherCount=0;
    var facebook_google_Count=0;
    var other_reports_send_config=5000;
    var google_facebook_reports_send_config=5000;
    current_site={
        id : filePath.split('=')[1].split('&')[0]
    };
    async.waterfall([
        function (cb) {
            readStream = fs.createReadStream(filePath);
            console.log('---test', filePath);
            readStream.pipe(csv())
                .on('data', function (data) {
                    //  readStream.pause();
                    //数据封装，打包
                    var media_source=data['Media Source'];
                    if (media_source==="googleadwords_int" || media_source==="Facebook Ads") {
                        item_fg=packItemForInAppEventReportFacebookGoogle(data);
                        if (item_fg.err) {
                            console.log('item_fg.err',item_fg.err);
                            return callback(null);
                        } else {
                            eventDataFilterFacebookGoogle(item_fg.value,google_facebook_reports);
                        }
                    } else {
                        item_other = packItemForInAppEventReport(data);
                        if (item_other.err) {
                            console.log('item_other.err', item_other.err);
                            return callback(null);
                        } else {
                            //数据重复过滤
                            eventDataFilter(item_other.value, other_reports);
                        }
                    }
                    tCount++;

                    //发送其余数据
                    async.waterfall([
                        function (fn) {
                            if (other_reports.length >= other_reports_send_config && dbFlag) {
                                console.log('on data... send before other_reports.length=',other_reports.length);
                                dbFlag = false;
                                readStream.pause();
                                var temOther = other_reports.slice(0, other_reports_send_config);
                                Dbapi.sendMsg({
                                    json: {
                                        actionid: 8220,
                                        noat: true,
                                        events: temOther
                                    }
                                }, function (err, result) {
                                    console.log('other_reports Dbapi sendMsg', 8220, err, result);
                                    getCount += temOther.length;
                                    otherCount+=temOther.length;
                                    console.log('on data other_reports tCount', tCount, 'getCount', getCount,'otherCount',otherCount);
                                    other_reports.splice(0, temOther.length);
                                    console.log('on data... after other_reports.length=',other_reports.length);
                                    dbFlag = true;
                                    readStream.resume();
                                    fn(err);
                                }, 2);
                            } else {
                                fn(null);
                            }
                        },
                        function (fn) {
                            //发送facebook，google数据
                            // console.log('google_facebook_reports.length=',google_facebook_reports.length);
                            if (google_facebook_reports.length >= google_facebook_reports_send_config && dbFlag) {
                                console.log('on data ...send before google_facebook_reports.length=',google_facebook_reports.length);
                                dbFlag = false;
                                readStream.pause();
                                //发送其余数据
                                var temp_google_facebook_ = google_facebook_reports.slice(0, google_facebook_reports_send_config);
                                Dbapi.sendEvent({
                                    json: {
                                        actionid: 1001,
                                        noat: true,
                                        events: temp_google_facebook_
                                    }
                                }, function (err, result) {
                                    console.log('google_facebook_reports Dbapi sendMsg', 1001, err, result);
                                    getCount += temp_google_facebook_.length;
                                    facebook_google_Count+=temp_google_facebook_.length;
                                    console.log('on data google_facebook_reports tCount', tCount, 'getCount', getCount,'facebook_google_Count',facebook_google_Count);
                                    google_facebook_reports.splice(0, temp_google_facebook_.length);
                                    console.log('on data... send after google_facebook_reports.length=',google_facebook_reports.length);
                                    dbFlag = true;
                                    readStream.resume();
                                    fn(err);
                                }, 2);
                            } else {
                                fn(null);
                            }
                        }
                    ],function (err) {
                        if (err) console.log('readInAppEventReports on(data) ',err);
                        if (tCount>=API_MAX) {
                            reports.push(data);
                        }
                    });
                })
                .on('end', function () {
                    var otherMsgSend=function (buff,cb) {
                        Dbapi.sendMsg({
                            json: {
                                actionid: 8220,
                                noat: true,
                                events: buff
                            }
                        }, function (err, result) {
                            console.log('other_reports Dbapi sendMsg', 8220, err, result);
                            getCount += buff.length;
                            otherCount+=buff.length;
                            console.log('other_reports *** tCount', tCount, 'getCount', getCount,'otherCount',otherCount);
                            cb(err);
                        }, 2);
                    };
                    var fgMsgSend=function (buff,cb) {
                        Dbapi.sendEvent({
                            json: {
                                actionid: 1001,
                                noat: true,
                                events: buff
                            }
                        }, function (err, result) {
                            console.log('google_facebook_reports Dbapi sendMsg', 1001, err, result);
                            getCount += buff.length;
                            facebook_google_Count+=buff.length;
                            console.log('google_facebook_reports *** tCount', tCount, 'getCount', getCount,'facebook_google_Count',facebook_google_Count);
                            cb(err);
                        }, 2);
                    };
                    async.waterfall([
                        function (fn) {
                            console.log('on end read other_reports');
                            console.log('on end send before other_reports.length =',other_reports.length );
                            if (other_reports.length !== 0) {
                                var flag=true;
                                async.whilst(
                                    function () {
                                        return flag;
                                    },
                                    function (fn_cb) {
                                        console.log('on end send..other_reports.length =',other_reports.length );
                                        if (other_reports.length > other_reports_send_config){
                                            flag=true;
                                            var buff=other_reports.splice(0,other_reports_send_config);
                                            otherMsgSend(buff,function (err) {
                                                fn_cb(err);
                                            });
                                        } else {
                                            flag=false;
                                            return fn_cb(null)
                                        }
                                    },
                                    function (err) {
                                        if (err) console.log(err);
                                        otherMsgSend(other_reports,function (error) {
                                            fn(error);
                                        });
                                    });
                            } else {
                                fn(null)
                            }
                        },
                        function (fn) {
                            console.log('on end read google_facebook_reports');
                            console.log('on end send before google_facebook_reports.length=',google_facebook_reports.length);
                            if (google_facebook_reports.length !==0) {
                                var flag=true;
                                async.whilst(
                                    function () {
                                        return flag;
                                    },
                                    function (fn_cb) {
                                        console.log('on end send...google_facebook_reports.length=',google_facebook_reports.length);
                                        if (google_facebook_reports.length > google_facebook_reports_send_config) {
                                            flag=true;
                                            var buff=google_facebook_reports.splice(0,google_facebook_reports_send_config);
                                            fgMsgSend(buff,function (err) {
                                                fn_cb(err);
                                            });
                                        } else {
                                            flag=false;
                                            return fn_cb(null);
                                        }
                                    },
                                    function (err) {
                                        if (err) console.log(err);
                                        fgMsgSend(google_facebook_reports,function (error) {
                                            fn(error);
                                        });
                                    });
                            } else {
                                fn(null);
                            }
                        }
                    ],function (err) {
                        if (err) console.log('readInAppEventReports on(end) ',err);
                        cb(null);
                    });
                });
        },
        function (cb) {
            // fs.unlink(filePath, function (err) {
            //  cb(err);
            // });
            cb(null);
        }
    ], function (err) {
        if (err) console.log(err);
        async.waterfall([
            function (fn) {
                console.log('tCount=',tCount,'API_MAX=',API_MAX);
                if (tCount >= API_MAX - 20000) {
                    console.log('readInAppEventReports API_MAX - 20000 Monitor ...');
                    var apiErr = Monitor.errFactory();
                    apiErr.err = '数据不完整';
                    apiErr.funct = readInstallsReport.toString();
                    apiErr.param = filePath;
                    apiErr.level = 10;
                    Monitor.send(apiErr);
                }
                fn(null);
            },
            function (fn) {
                console.log('tCount=',tCount,'API_MAX=',API_MAX);
                if (tCount >= API_MAX) {
                    console.log('readInAppEventReports data >',API_MAX/1000,'K');
                    // console.log('reports.length=',reports.length);
                    // console.log(JSON.stringify(reports[0]));
                    // var str='2017/7/10  23:59:59';
                    var detail_date=[];
                    detail_date=reports[0]['Event Time'].substr(10,6).split(':');
                    detail_date[0]=detail_date[0].replace(" ","");
                    if (parseInt(detail_date[1])<59) {
                        detail_date[1]=parseInt(detail_date[1])+1;
                    }
                    if (parseInt(detail_date[1])===59) {
                        detail_date[0]=parseInt(detail_date[0])+1;
                        detail_date[1]='00';
                    }
                    console.log('detail_date:',detail_date);
                    var param={
                        detail_date:detail_date,
                        website:current_site
                    };
                    setTimeout(function () {
                        inAppEventReport(user, date, param, function (err) {
                            console.log('inAppEventReport end err', err);
                            Monitor.send('read event report');
                            fn(err);
                        });
                    },3000);

                } else if (tCount===6 || other_reports.length===0 || google_facebook_reports.length===0) {
                    console.log('当前时间',new Date().toLocaleDateString(),'待获取应用日期 date=',date,' website= ',current_site.id,'应用超过请求次数');
                    fn(null);
                } else {
                    console.log('当前时间',new Date().toLocaleDateString(),'待获取应用日期 date=',date,' website= ',current_site.id,'该天没有数据');
                    fn(null);
                }
            }
        ],function (err) {
            callback(err);
        });
    });
}

集合遍历，eachLimit里面的方法会并行
            async.eachLimit(users[user].websites, users[user].websites.length, function (website, cb) {
                console.log('inAppEventReports user=',user,'date=',date,'website=',website);
                loadInAppEventReports(user, website, date, function (err, path) {
                    if (path) filesPath = filesPath.concat(path);
                    console.log('loadInAppEventReports end', website.id, 'err', err, new Date().toString());
                    cb(null);
                })
            }, function (err) {
                cba(err, filesPath);
            });
可以将limit设置为1 使其串行
    async.eachLimit(dateList.slice(0,6), 1, function (dateItem, callback) {
        console.log('waterfall-start', dateItem);
        async.waterfall([
            function (cb) {
                mainFunction(dateItem, dateItem, function (err) {
                    cb(null);
                });
            }
        ], function (err) {
            callback(null);
        });
    }, function (err) {
        console.error('history end err', err);
    });