先介绍下多路归并算法：
多路归并是外部排序(External Sort)的基础，实现也比较简单，和最简单的归并排序中的二路归并是基本一样的，只不过路数是浮动的k。
(1)假设有K路数据流，流内部是有序的，且流间同为升序或降序
(2)首先读取每个流的第一个数，如果已经EOF，pass
(3)将有效的k(k可能小于K)个数比较，选出最小的那路mink，输出，读取mink的下一个
(4)直到所有K路都EOF
这个代码的背景是：需要对表进行汇总统计，但是表数据太大，无法通过group by进行sum统计，所以需要把表数据拆分读出到N份文件中，对这N份文件进行多路归并排序。最后把生成的文件通过sqlload入库。这里不会进行delete表操作，而是通过drop table、create table的形式进行装入，并建立索引。
这个代码还实现了增量的更新，避免每天都需要对月表进行全量统计。只需要保存一个昨天的数据跟今天的数据进行比较。
下面介绍下这个代码：
头文件：
// Created: 2013-04-27
// Module: 量收系统号码收入日累计汇总程序
// Author: jcmozw
// Revision: version1.1
// Description: 汇总量收系统从月初1号到当前时间前一天的号码累计收入
//
#ifndef DAYBILLMONTHCHARGE_H_
#define DAYBILLMONTHCHARGE_H_
#include "DaybillLog.h"
#include "dbutil/DBUtil.h"
#include "Poco/Util/Application.h"
#include "Poco/Util/Option.h"
#include "Poco/Util/OptionSet.h"
#include "Poco/Util/HelpFormatter.h"
#include "Poco/Util/AbstractConfiguration.h"
#include "Poco/Util/Validator.h"
#include "Poco/Util/RegExpValidator.h"
#include "Poco/NumberFormatter.h"
#include "Poco/NumberParser.h"
#include "Poco/String.h"
#include "Poco/Exception.h"
#include "Poco/Foundation.h"
#include "Poco/Environment.h"
#include "Poco/StringTokenizer.h"
#include "Poco/DateTimeFormatter.h"
#include "Poco/DateTimeFormat.h"
#include "Poco/LocalDateTime.h"
#include "Poco/Timespan.h"
#include "Poco/File.h"
#include "Poco/Glob.h"
#include "Poco/Path.h"
#include<algorithm>
#include <vector>
#include <set>
#include<deque>
#include<fstream>
#include <iostream>
#include <sys/wait.h>
#include <assert.h>
#include <unistd.h>
#include <sys/types.h>
#include <signal.h>
#include <errno.h>
using Poco::Util::Application;
using Poco::Util::Option;
using Poco::Util::OptionSet;
using Poco::Util::HelpFormatter;
using Poco::Util::AbstractConfiguration;
using Poco::Util::OptionCallback;
using Poco::toLower;
using Poco::File;
using Poco::Glob;
using Poco::Path;
using Poco::LocalDateTime;
using Poco::DateTimeFormatter;
using Poco::DateTimeFormat;
using Poco::Timespan;
using std::vector;
using std::set;
using std::deque;
static const int FORKNUM= 5; ///进程数
static const int YFFPAYMENT=1;//预付费标识
static const int HFFPAYMENT=2;//后付费标识
static const int READBILLTABLE=1;//读收入A表标识
static const int READSUMTABLE=2;//读汇总表标识
static const int MERGEFILE=3;//归并当前账期结果文件(正常流程)
static const int MERGEINCREMENT=4;//归并增量文件
static const int MERGEFILESPECIAL=5;//调用存储过程后把汇总表读出归并到昨天文件供下一天进行增量处理(非正常流程)
static const int SQLLOADHFFBILL=1;//使用sqlload读取后付费原始文件入库
static const int SQLLOADHFFGY=2;//使用sqlload读取后付费公允文件入库
static const int SQLLOADYFF=3;//使用sqlload读取预付费公允文件入库
static const int HFFBILLDATASRCTYPE=101;//后付费据源101
static const int HFFGYDATASRCTYPE=104;//后付费公允数据源104
类MonthChargeLink
class MonthChargeLink{
public:
int _file_number;
string _acc_nbr;
string _serv_nbr;
int _corp_id;
int _prod_id;
int _acct_item_type_id;
int _data_src_type;
int _data_src_detail_type;
long int _serv_id;
long _charge;
MonthChargeLink(){};
MonthChargeLink(int icorpid,long int liservid,string straccnbr,string strservnbr,int iprod_id,int iacctitemtype_id,int idatasrctype,int idatasrcdetailtype,long lcharge){
this->_corp_id=icorpid;
this->_serv_id=liservid;
this->_acc_nbr=straccnbr;
this->_serv_nbr=strservnbr;
this->_prod_id=iprod_id;
this->_acct_item_type_id=iacctitemtype_id;
this->_data_src_type=idatasrctype;
this->_data_src_detail_type=idatasrcdetailtype;
this->_charge=lcharge;
}
MonthChargeLink(int ifilenumber,int icorpid,long int liservid,string straccnbr,string strservnbr,int iprod_id,int iacctitemtype_id,int idatasrctype,int idatasrcdetailtype,long lcharge){
this->_file_number=ifilenumber;
this->_corp_id=icorpid;
this->_serv_id=liservid;
this->_acc_nbr=straccnbr;
this->_serv_nbr=strservnbr;
this->_prod_id=iprod_id;
this->_acct_item_type_id=iacctitemtype_id;
this->_data_src_type=idatasrctype;
this->_data_src_detail_type=idatasrcdetailtype;
this->_charge=lcharge;
}
MonthChargeLink& operator=(const MonthChargeLink& obj) {
_file_number=obj._file_number;
_corp_id=obj._corp_id;
_acc_nbr=obj._acc_nbr;
_serv_nbr=obj._serv_nbr;
_serv_id=obj._serv_id;
_data_src_type=obj._data_src_type;
_data_src_detail_type=obj._data_src_detail_type;
_acct_item_type_id=obj._acct_item_type_id;
_prod_id=obj._prod_id;
_charge=obj._charge;
return *this;
}
bool operator<( const MonthChargeLink& obj ) const {
if (_corp_id != obj._corp_id){
return _corp_id < obj._corp_id;
} else if (_serv_id != obj._serv_id){
return _serv_id < obj._serv_id;
} else if (_acc_nbr != obj._acc_nbr){
return _acc_nbr < obj._acc_nbr;
} else if (_serv_nbr != obj._serv_nbr){
return _serv_nbr < obj._serv_nbr;
} else if (_prod_id != obj._prod_id){
return _prod_id < obj._prod_id;
}else if (_acct_item_type_id != obj._acct_item_type_id){
return _acct_item_type_id < obj._acct_item_type_id;
}else if (_data_src_type != obj._data_src_type){
return _data_src_type < obj._data_src_type;
}else if (_data_src_detail_type != obj._data_src_detail_type){
return _data_src_detail_type < obj._data_src_detail_type;
}
return 0;
}
bool operator==(const MonthChargeLink& obj) const {
if (_corp_id== obj._corp_id
&&_data_src_type== obj._data_src_type
&&_data_src_detail_type== obj._data_src_detail_type
&&_acc_nbr==obj._acc_nbr
&&_serv_nbr==obj._serv_nbr
&&_prod_id== obj._prod_id
&&_serv_id== obj._serv_id
&&_acct_item_type_id== obj._acct_item_type_id) {
return 1;
}
return 0;
}
~MonthChargeLink(){}
};
文件类：
class File_Item
{
public:
std::ifstream *_readFile;//读文件流对象
int _file_number;//文件编码
string _file_name;//文件名称
bool _read_next;
File_Item(int ifilenumber,string strfilename,bool blreadnext,std::ifstream *ifsreadFile){
this->_file_number=ifilenumber;
this->_file_name=strfilename;
this->_read_next=blreadnext;
this->_readFile=ifsreadFile;
}
};
比较类
class DayBillMonthCmp//比较类
{
public:
bool operator()( const MonthChargeLink*left, const MonthChargeLink* right )const
{
return *left < *right;
}
};
DayBillMonthCharge类
class DayBillMonthCharge:public Application{
public:
DayBillMonthCharge(){
_helpRequested=false;
_log_dir="";
_log_name="";
_corp_head="";
_billing_day="";
_filedir="";
_yesterday="";
_last_date="";
_payment_type=-1;
_configdir="";
_result_db_table="";
_result_yddb_table="";
_result_yddbsz_table="";
_logtext="";
}
bool ForkMain(int iforktype);
void ForkProcess(int imod,int iforktype);//启动后付费多进程处理
void ForkYffProcess(int mod);//启动预付费多进程处理
bool ReadHffTableProce(int imod,int itabletype);//处理后付费数据
bool ReadYffTableProce(int imod,int itabletype);//处理预付费数据
bool ReadDayServProcess(int imod,int itabletype);//处理累计月数据
void writeBillFile(string strfilename);
void mergePatternFiles(string strsourcename,string strtargetname);
void getPatternFilesMsg(string strfilename);//获取进程级文件
void mergeProcess(string file_name);//归并文件数据
void readFirstRecord();//取N份文件的首行记录
bool SortRecord();//取出的N份文件的首行记录进行排序
bool AddSpecialFilesMsg(string strfilename,int ifileorder);
bool mergeModSumFilesProc(int imod);
void mergeYffFinalResultProc();
bool mergeResultProc(int imod);
bool mergeServSumFilesProc(int imod);
void writeMergeFile(std::ofstream &o_file);
void removeFile(string strfilename);
void ReNameFile(string strsourcename,string strdesname);
bool sqlloadDate(int iinserttype);
void creatTableIndex();
void creatTable();
void insertDate();
protected:
void initialize( Application& self ){
Application::initialize( self );
}
void uninitialize() {
Application::uninitialize();
}
void reinitialize( Application& self ) {
Application::reinitialize( self );
}
void defineOptions( OptionSet& options ) {
Application::defineOptions(options);
options.addOption(
Option("help", "h", "显示帮助信息").required(false).repeatable(false).callback(
OptionCallback < DayBillMonthCharge> (this, &DayBillMonthCharge::handleHelp)));
options.addOption(
Option("billingid", "d", "日期").required(true).repeatable(false).argument("billingid").callback(
OptionCallback < DayBillMonthCharge> (this, &DayBillMonthCharge::handleBillingday)));
options.addOption(
Option("corpid", "l", "分公司").required(true).repeatable(false).argument("corpid").callback(
OptionCallback < DayBillMonthCharge> (this, &DayBillMonthCharge::handlecorpid)));
Poco::Util::Validator* paymentid = new Poco::Util::RegExpValidator("1|2");
options.addOption(
Option("paymentid", "p", "付费类型:1预付费,2后付费").required(true).repeatable(false).argument("paymentid").callback(
OptionCallback < DayBillMonthCharge> (this, &DayBillMonthCharge::handlepayment)));
}
void handlecorpid( const std::string&name, const std::string& value ) {
config().setString( "application.company", value );
_corp_head = value;
}
void handleBillingday( const std::string&name, const std::string& value ) {
_billing_day = value;
}
void handlepayment( const std::string&name, const std::string& value ) {
_payment_type = atoi(value.c_str());
}
void handleHelp( const std::string& name, const std::string& value ) {
_helpRequested = true;
displayHelp();
stopOptionsProcessing();
}
void displayHelp() {
HelpFormatter helpFormatter( options() );
helpFormatter.setCommand( commandName() );
helpFormatter.setWidth( 120 );
helpFormatter.setUsage( "OPTIONS" );
helpFormatter.setHeader( "DayBillMonthCharge" );
helpFormatter.format( std::cout );
}
int main(const std::vector<std::string>& args);
private:
bool _helpRequested;//是否帮忙标识--h
string _log_dir;//日志目录
string _logtext;//日志内容
string _log_name;//日志名称
string _filedir;//数据文件目录
string _configdir;//配置目录
std::string _corp_head;//分公司
std::string _billing_day;//账期
int _payment_type;//付费类型
string _result_db_table;//号码汇总结果表
string _result_yddb_table;//移动汇总结果表
string _result_yddbsz_table;//深圳移动汇总结果表(新增)
string _last_date;//账期的前一天日期
string _yesterday;//当前系统的前一天日期
set<int>file_num_set;//保存链表中相同记录的文件编号
typedef set<int>::iterator file_num_iter;
vector<File_Item>file_item_vector;//记录文件状态信息，控制文件读取
typedef vector<File_Item>::iterator file_item_Iter;
deque<MonthChargeLink*>m_dequeMonthCharge;
typedef deque<MonthChargeLink*>::iterator m_dequeMonthChargeIter;
vector<MonthChargeLink*>m_vectorMonthCharge;//保存结果容器
typedef vector<MonthChargeLink*>::iterator m_vectorMonthChargeIter;
set<MonthChargeLink*, DayBillMonthCmp > m_setMonthCharge;//保存库表读出的数据
typedef set<MonthChargeLink*, DayBillMonthCmp >::iterator m_setMonthChargeIter;
};
POCO_APP_MAIN(DayBillMonthCharge)
#endif /* DAYBILLMONTHCHARGE_H_ */
源文件
// Created: 2013-04-27
// Module: 量收系统号码收入日累计汇总程序
// Author: jcmozw
// Revision: version1.1
// Description: 汇总量收系统从月初1号到当前时间前一天的号码累计收入
//
#include "DayBillMonthCharge.h"
pid_t r_wait(int *stat_loc) {
int retval;
while (((retval = wait(stat_loc)) == -1) && (errno == EINTR))
;
return retval;
}
pid_t* childpid = NULL;
extern "C" {
void sig_int(int signo) {
if (getpid() == getpgid(0)) {
for (int i = 0;
*(childpid + i) != 0 || i < sizeof(childpid) / sizeof(pid_t);
i++) {
if (*(childpid + i) != 0) {
kill(*(childpid + i), SIGKILL);
}
}
while (wait(NULL) > 0) {
;
}
exit(0);
}
}
}
bool DayBillMonthCharge::ForkMain(int iforktype){
DayBillLog * daylog = new DayBillLog(_corp_head);
int iforknum=0;
if((iforktype==READBILLTABLE)&&(_payment_type==YFFPAYMENT)){
iforknum=3;
}else if((iforktype==READBILLTABLE)&&(_payment_type==HFFPAYMENT)){
iforknum=2;
}else if((iforktype==READSUMTABLE)&&(_payment_type==YFFPAYMENT)){
iforknum=1;
}else if((iforktype==READSUMTABLE)&&(_payment_type==HFFPAYMENT)){
iforknum=2;
}else if((iforktype==MERGEFILE)&&(_payment_type==HFFPAYMENT)){
iforknum=2;
}else if((iforktype==MERGEFILE)&&(_payment_type==YFFPAYMENT)){
iforknum=3;
}else if((iforktype==MERGEINCREMENT)&&(_payment_type==HFFPAYMENT)){
iforknum=2;
}else if((iforktype==MERGEINCREMENT)&&(_payment_type==YFFPAYMENT)){
iforknum=1;
}else if((iforktype==MERGEFILESPECIAL)&&(_payment_type==HFFPAYMENT)){
iforknum=2;
}else if((iforktype==MERGEFILESPECIAL)&&(_payment_type==YFFPAYMENT)){
iforknum=1;
}else {
return false;
}
signal(SIGINT, sig_int);
childpid = new pid_t[iforknum];
for (int i = 0; i < iforknum; i++) {
childpid[i] = fork();
if (childpid[i] == -1) {
_logtext = "ForkMain fork failed for"+Poco::NumberFormatter::format(strerror(errno));
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
exit(1);
} else if (childpid[i] == 0) {
try {
if(iforktype==READBILLTABLE||iforktype==READSUMTABLE){
ForkProcess(i,iforktype);
}else if(iforktype==MERGEFILE){
mergeModSumFilesProc(i);
}else if(iforktype==MERGEINCREMENT){
mergeResultProc(i);
}else if(iforktype==MERGEFILESPECIAL){
mergeServSumFilesProc(i);
}
} catch (Poco::Exception& ex) {
_logtext = "ForkMain启动进程失败，错误信息:"+ex.displayText();
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
exit(0);
}
}
等待所有的子进程结束
int status = 0;
pid_t retPid;
while ((retPid = r_wait(&status)) > 0) {
if (WIFEXITED(status)) {
_logtext = "ForkMain子进程["+Poco::NumberFormatter::format(retPid)+"] 退出,状态为"+Poco::NumberFormatter::format(WEXITSTATUS(status));
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
} else if (WIFSIGNALED(status)) {
_logtext = "ForkMain子进程["+Poco::NumberFormatter::format(retPid)+"] killed 信号为"+Poco::NumberFormatter::format(WTERMSIG(status));
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
} else if (WIFSTOPPED(status)) {
_logtext = "ForkMain子进程["+Poco::NumberFormatter::format(retPid)+"] stopped 信号为"+Poco::NumberFormatter::format(WSTOPSIG(status));
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
} else {
_logtext = "ForkMain子进程["+Poco::NumberFormatter::format(retPid)+"] 退出";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
}
return true;
}
void DayBillMonthCharge::ForkProcess(int imod,int iforktype){
DayBillLog * daylog = new DayBillLog(_corp_head);
int process = FORKNUM; ///并发进程个数
string _logtext;
signal(SIGINT, sig_int);
childpid = new pid_t[process];
for (int i = 0; i < process; i++) {
childpid[i] = fork();
if (childpid[i] == -1) {
_logtext = "ForkProcess fork failed for"+Poco::NumberFormatter::format(strerror(errno));
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
exit(1);
} else if (childpid[i] == 0) {
try {
if((iforktype==READBILLTABLE)&&(_payment_type==HFFPAYMENT)){
ReadHffTableProce(i,imod);
}else if((iforktype==READBILLTABLE)&&(_payment_type==YFFPAYMENT)){
ReadYffTableProce(i,imod);
}else if(iforktype==READSUMTABLE){
ReadDayServProcess(i,imod);
}
} catch (Poco::Exception& ex) {
_logtext = "ForkProcess 启动进程失败，错误信息:"+ex.displayText();
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
exit(0);
}
}
等待所有的子进程结束
int status = 0;
pid_t retPid;
while ((retPid = r_wait(&status)) > 0) {
if (WIFEXITED(status)) {
_logtext = "ForkProcess子进程["+Poco::NumberFormatter::format(retPid)+"] 退出,状态为"+Poco::NumberFormatter::format(WEXITSTATUS(status));
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
} else if (WIFSIGNALED(status)) {
_logtext = "ForkProcess子进程["+Poco::NumberFormatter::format(retPid)+"] killed 信号为"+Poco::NumberFormatter::format(WTERMSIG(status));
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
} else if (WIFSTOPPED(status)) {
_logtext = "ForkProcess子进程["+Poco::NumberFormatter::format(retPid)+"] stopped 信号为"+Poco::NumberFormatter::format(WSTOPSIG(status));
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
} else {
_logtext = "ForkProcess子进程["+Poco::NumberFormatter::format(retPid)+"] 退出";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
}
}
bool DayBillMonthCharge::ReadHffTableProce(int imod,int itabletype){
DayBillLog * daylog = new DayBillLog(_corp_head);
string strsql, straccnbr,strdatatable,strfilename,strsourname,strtargetname;
int icorpidone,icorpid,iprodid,iacctitemtypeid,idatasrctype;
int imodorder=1;
long int liservid=-1;
long lcharge=0;
long lcount=0;
if(itabletype==0){
strdatatable=" daybill_hf_"+_corp_head+".if_sure_src_day_bill_" + _billing_day.substr(0, 6);
idatasrctype=HFFBILLDATASRCTYPE;
strsourname=_filedir + _corp_head + "_daybill_serv_hfbill_"+ _billing_day + "_"+Poco::NumberFormatter::format(imod)+"_*.txt";
strtargetname=_filedir + _corp_head + "_daybill_serv_hfbill_"+ _billing_day +"_"+Poco::NumberFormatter::format(imod)+".txt";
}else if(itabletype==1){
idatasrctype=HFFGYDATASRCTYPE;
strdatatable=" daybill_hf_"+_corp_head+".if_sure_src_day_gy_" + _billing_day.substr(0, 6);
strsourname=_filedir + _corp_head + "_daybill_serv_hfgy_"+ _billing_day + "_"+Poco::NumberFormatter::format(imod)+"_*.txt";
strtargetname=_filedir + _corp_head + "_daybill_serv_hfgy_"+ _billing_day + "_"+Poco::NumberFormatter::format(imod)+".txt";
}else{
return false;
}
m_setMonthCharge.clear();
DBUtil::DBConnectionFactory& factory =DBUtil::DBConnectionFactory::getInstance();
DBUtil::DBConnection* conn = factory.createConnection(_corp_head, true);
conn->open();
try{
strsql="select corp_id from daybill_report.dic_corp where lower(corp_head)=lower('"+_corp_head+"')";
DBUtil::Statement st = conn->createStatement(strsql);
st.execute();
st>>icorpidone;
strsql =" select serv_id,acc_nbr,prod_id,acct_item_type_id,charge";
strsql += " from "+strdatatable+" a ";
strsql += " where billing_cycle_id=" + _billing_day;
strsql += " and ABS(MOD(TRUNC(out_seq_id/200), "+Poco::NumberFormatter::format(FORKNUM)+")) = "+Poco::NumberFormatter::format(imod);
st = conn->createStatement(strsql);
st.execute();
while (st >> liservid >> straccnbr >> iprodid >> iacctitemtypeid >> lcharge) {
if (++lcount % 200000 == 0){
_logtext = "进程号["+Poco::NumberFormatter::format(imod)+"]已读取"+strdatatable+"记录"+Poco::NumberFormatter::format(lcount);
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
MonthChargeLink * monthclass=new MonthChargeLink(icorpidone,liservid,straccnbr,"-1",iprodid,iacctitemtypeid,idatasrctype,idatasrctype,lcharge);
std::pair<m_setMonthChargeIter, bool> res = m_setMonthCharge.insert(monthclass);
if (!res.second) {
(*(res.first))->_charge += lcharge;
delete monthclass;
monthclass = NULL;
}
if (m_setMonthCharge.size() >= 200000){
if(itabletype==0){
strfilename=_filedir + _corp_head + "_daybill_serv_hfbill_"+ _billing_day + "_"+Poco::NumberFormatter::format(imod)+"_"+Poco::NumberFormatter::format(imodorder)+".txt";
}else if(itabletype==1){
strfilename=_filedir + _corp_head + "_daybill_serv_hfgy_"+ _billing_day + "_"+Poco::NumberFormatter::format(imod)+"_"+Poco::NumberFormatter::format(imodorder)+".txt";
}
writeBillFile(strfilename);
imodorder++;
}
}
}catch (DBUtil::SQLException& ex) {
_logtext = "进程号["+Poco::NumberFormatter::format(imod)+"]已读取"+strdatatable+"表异常，异常信息:"+ex.what();
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}catch (Poco::Exception& ex) {
_logtext = "进程号["+Poco::NumberFormatter::format(imod)+"]已读取"+strdatatable+"表异常，异常信息:"+ ex.displayText();
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}catch(...){
_logtext = "进程号["+Poco::NumberFormatter::format(imod)+"]已读取"+strdatatable+"表异常，异常信息:其他异常";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
if(itabletype==0){
strfilename=_filedir + _corp_head + "_daybill_serv_hfbill_"+ _billing_day + "_"+Poco::NumberFormatter::format(imod)+"_"+Poco::NumberFormatter::format(imodorder)+".txt";
}else if(itabletype==1){
strfilename=_filedir + _corp_head + "_daybill_serv_hfgy_"+ _billing_day + "_"+Poco::NumberFormatter::format(imod)+"_"+Poco::NumberFormatter::format(imodorder)+".txt";
}
writeBillFile(strfilename);
conn->close();
_logtext = "进程号["+Poco::NumberFormatter::format(imod)+"]读取"+strdatatable+"结束,总记录"+Poco::NumberFormatter::format(lcount);
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
mergePatternFiles(strsourname,strtargetname);
return true;
}
bool DayBillMonthCharge::SortRecord(){
DayBillMonthCharge *s;
int list_size=m_dequeMonthCharge.size();
if(list_size>0){
file_num_set.clear();
if(list_size>1){
sort(m_dequeMonthCharge.begin(),m_dequeMonthCharge.end(),DayBillMonthCmp());
file_num_set.insert(m_dequeMonthCharge.front()->_file_number);
for(int i=0;i<m_dequeMonthCharge.size()-1;i++){
if(*(m_dequeMonthCharge[i])==*(m_dequeMonthCharge[i+1])){
file_num_set.insert(m_dequeMonthCharge[i+1]->_file_number);
}else{
break;
}
}
}else if(list_size>0){
file_num_set.insert(m_dequeMonthCharge.front()->_file_number);
}
for(file_item_Iter file_item_iter = file_item_vector.begin();file_item_iter != file_item_vector.end(); file_item_iter++){
file_item_iter->_read_next=false;
for(file_num_iter fiter=file_num_set.begin();fiter!=file_num_set.end();fiter++){
if(file_item_iter->_file_number==*fiter){
file_item_iter->_read_next=true;
break;
}
}
}
}else{
return false;
}
return true;
}
void DayBillMonthCharge::mergeProcess(string strfilename){
DayBillLog * daylog = new DayBillLog(_corp_head);
bool blifrun=true;
int icorpid;
long int liservid;
string straccnbr;
string strservnbr;
int iprodid;
int iacctitemtypeid;
int idatasrctype;
int idatasrcdetailtype;
long lcharesum=0;
long lcount=0;
int ifileitemsize=file_item_vector.size();
std::ofstream o_file;
o_file.open(strfilename.c_str());
m_vectorMonthCharge.clear();
m_dequeMonthCharge.clear();
int ipopnum=0;
int ipopsum=0;
do {
readFirstRecord();
blifrun=SortRecord();
if(blifrun){
icorpid=m_dequeMonthCharge.front()->_corp_id;
liservid=m_dequeMonthCharge.front()->_serv_id;
straccnbr=m_dequeMonthCharge.front()->_acc_nbr;
strservnbr=m_dequeMonthCharge.front()->_serv_nbr;
iprodid=m_dequeMonthCharge.front()->_prod_id;
iacctitemtypeid=m_dequeMonthCharge.front()->_acct_item_type_id;
idatasrctype=m_dequeMonthCharge.front()->_data_src_type;
idatasrcdetailtype=m_dequeMonthCharge.front()->_data_src_detail_type;
lcharesum=0;
ipopsum=file_num_set.size();
ipopnum=0;
while (!m_dequeMonthCharge.empty()){
if(ipopnum<ipopsum){
lcharesum+=m_dequeMonthCharge.front()->_charge;
MonthChargeLink *deletenode=m_dequeMonthCharge.front();//指向数据堆指针
delete deletenode;//释放指针指向的数据堆
deletenode=NULL;//对释放掉的指针设置为空指针
m_dequeMonthCharge.pop_front();
}else{
break;
}
ipopnum++;
}
if (++lcount % 500000 == 0) {
_logtext = "已归并"+strfilename+"记录"+Poco::NumberFormatter::format(lcount);
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
MonthChargeLink * monthclass=new MonthChargeLink(icorpid,liservid,straccnbr,strservnbr,iprodid,iacctitemtypeid,idatasrctype,idatasrcdetailtype,lcharesum);
m_vectorMonthCharge.push_back(monthclass);
if (m_vectorMonthCharge.size()>=200000){
writeMergeFile(o_file);
}
}
} while (blifrun);
writeMergeFile(o_file);
m_dequeMonthCharge.clear();
o_file.close();
_logtext = "归并到文件"+strfilename+"结束，总记录数是："+Poco::NumberFormatter::format(lcount)+"条记录";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
void DayBillMonthCharge::readFirstRecord(){
char buffer[1024] = { 0 };
memset(buffer, '\0', 1024);
int icorpid,idatasrctype,idatasrcdetailtype,iprodid,iacctitemtypeid;
string straccnbr,strservnbr;
long int liservid = 0;
long lcharge = 0;
string strdelim = "|";
string strreadLine = "";
int ifilenumber=0;
std::ifstream *readFile;
for (file_item_Iter file_item_iter = file_item_vector.begin();file_item_iter != file_item_vector.end(); file_item_iter++) {
ifilenumber=file_item_iter->_file_number;
readFile = file_item_iter->_readFile;
while ((file_item_iter->_read_next)&&(readFile->peek()!=EOF)){
memset(buffer, '\0', 1024);
readFile->getline(buffer, 1024, '\n');
strreadLine.assign(buffer);
size_t pos1 = strreadLine.find(strdelim);
icorpid= Poco::NumberParser::parse(Poco::trim(strreadLine.substr(0, pos1)));
size_t pos2 = strreadLine.find(strdelim, pos1 + strdelim.size());
liservid=Poco::NumberParser::parse64(Poco::trim(strreadLine.substr(pos1 + strdelim.size(),pos2 - pos1 - strdelim.size())));
size_t pos3 = strreadLine.find(strdelim, pos2 + strdelim.size());
straccnbr = strreadLine.substr(pos2 + strdelim.size(),pos3 - pos2 - strdelim.size());
size_t pos4 = strreadLine.find(strdelim, pos3 + strdelim.size());
strservnbr = strreadLine.substr(pos3 + strdelim.size(),pos4 - pos3 - strdelim.size());
size_t pos5 = strreadLine.find(strdelim, pos4 + strdelim.size());
iprodid = Poco::NumberParser::parse(Poco::trim(strreadLine.substr(pos4 + strdelim.size(),pos5 - pos4 - strdelim.size())));
size_t pos6 = strreadLine.find(strdelim, pos5 + strdelim.size());
iacctitemtypeid = Poco::NumberParser::parse(Poco::trim(strreadLine.substr(pos5 + strdelim.size(),pos6 - pos5 - strdelim.size())));
size_t pos7 = strreadLine.find(strdelim, pos6 + strdelim.size());
idatasrctype = Poco::NumberParser::parse(Poco::trim(strreadLine.substr(pos6 + strdelim.size(),pos7 - pos6 - strdelim.size())));
size_t pos8 = strreadLine.find(strdelim, pos7 + strdelim.size());
idatasrcdetailtype = Poco::NumberParser::parse(Poco::trim(strreadLine.substr(pos7 + strdelim.size(),pos8 - pos7 - strdelim.size())));
lcharge = Poco::NumberParser::parse64(Poco::trim(Poco::trim(strreadLine.substr(pos8 + strdelim.size()))));
MonthChargeLink * LinkNode=new MonthChargeLink(ifilenumber,icorpid,liservid,straccnbr,strservnbr,iprodid,iacctitemtypeid,idatasrctype,idatasrcdetailtype,lcharge);
m_dequeMonthCharge.push_back(LinkNode);
break;
}//end while (readFile.peek()!=EOF)
}// end
}
void DayBillMonthCharge::mergePatternFiles(string strsourcename,string strtargetname){
getPatternFilesMsg(strsourcename);
mergeProcess(strtargetname);
removeFile(strsourcename);
}
bool DayBillMonthCharge::mergeServSumFilesProc(int imod){
string strsourcename,strtargetname;
if((_payment_type==HFFPAYMENT)&&(imod==0)){
strsourcename=_filedir + _corp_head + "_daybill_serv_hfbill_"+ _yesterday +"_*.txt";
strtargetname=_filedir + _corp_head + "_daybill_serv_hfbill_"+ _yesterday +".txt";
}else if((_payment_type==HFFPAYMENT)&&(imod==1)){
strsourcename=_filedir + _corp_head + "_daybill_serv_hfgy_"+ _yesterday + "_*.txt";
strtargetname=_filedir + _corp_head + "_daybill_serv_hfgy_"+ _yesterday +".txt";
}else if((_payment_type==YFFPAYMENT)&&(imod==0)){
strsourcename=_filedir + _corp_head + "_daybill_serv_yf_"+ _yesterday +"_*.txt";
strtargetname=_filedir + _corp_head + "_daybill_serv_yf_"+ _yesterday +".txt";
}else{
return false;
}
mergePatternFiles(strsourcename,strtargetname);
return true;
}
void DayBillMonthCharge::mergeYffFinalResultProc(){
DayBillLog * daylog = new DayBillLog(_corp_head);
string _logtext;
file_item_vector.clear();
bool blifsuccess=false;
bool blifbillsuccess=false;
bool blifgysuccess=false;
bool blifotsuccess=false;
string strlastfilename,strfilename,strtargetname,stryfbillfile,stryfgyfile,stryfotfile;
stryfbillfile=_filedir + _corp_head + "_daybill_serv_yfbill_"+ _billing_day + ".txt";
blifbillsuccess=AddSpecialFilesMsg(stryfbillfile,1);
stryfgyfile=_filedir + _corp_head + "_daybill_serv_yfgy_"+ _billing_day + ".txt";
blifgysuccess=AddSpecialFilesMsg(stryfgyfile,2);
stryfotfile=_filedir + _corp_head + "_daybill_serv_yfot_"+ _billing_day + ".txt";
blifotsuccess=AddSpecialFilesMsg(stryfotfile,3);
strtargetname=_filedir + _corp_head + "_daybill_serv_yf_"+ _billing_day +".txt";
blifsuccess=blifbillsuccess&&blifgysuccess&&blifotsuccess;
if(blifsuccess){
mergeProcess(strtargetname);
removeFile(stryfbillfile);
removeFile(stryfgyfile);
removeFile(stryfotfile);
}else{
_logtext = "文件缺失，请查看文件是否存在";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
}
void DayBillMonthCharge::ReNameFile(string strsourcename,string strdesname){
File tmpDir(strsourcename);
if(tmpDir.exists()){
tmpDir.renameTo(strdesname);
}
}
void DayBillMonthCharge::removeFile(string strfilename){
set<string> fileset;
Glob::glob(strfilename, fileset);
set<std::string>::iterator it = fileset.begin();
for(;it!=fileset.end();++it){
File removefile(*it);
if(removefile.exists()){
removefile.remove(false);
}
}
}
sqlload入库
void DayBillMonthCharge::readFirstRecord(){
char buffer[1024] = { 0 };
memset(buffer, '\0', 1024);
int icorpid,idatasrctype,idatasrcdetailtype,iprodid,iacctitemtypeid;
string straccnbr,strservnbr;
long int liservid = 0;
long lcharge = 0;
string strdelim = "|";
string strreadLine = "";
int ifilenumber=0;
std::ifstream *readFile;
for (file_item_Iter file_item_iter = file_item_vector.begin();file_item_iter != file_item_vector.end(); file_item_iter++) {
ifilenumber=file_item_iter->_file_number;
readFile = file_item_iter->_readFile;
while ((file_item_iter->_read_next)&&(readFile->peek()!=EOF)){
memset(buffer, '\0', 1024);
readFile->getline(buffer, 1024, '\n');
strreadLine.assign(buffer);
size_t pos1 = strreadLine.find(strdelim);
icorpid= Poco::NumberParser::parse(Poco::trim(strreadLine.substr(0, pos1)));
size_t pos2 = strreadLine.find(strdelim, pos1 + strdelim.size());
liservid=Poco::NumberParser::parse64(Poco::trim(strreadLine.substr(pos1 + strdelim.size(),pos2 - pos1 - strdelim.size())));
size_t pos3 = strreadLine.find(strdelim, pos2 + strdelim.size());
straccnbr = strreadLine.substr(pos2 + strdelim.size(),pos3 - pos2 - strdelim.size());
size_t pos4 = strreadLine.find(strdelim, pos3 + strdelim.size());
strservnbr = strreadLine.substr(pos3 + strdelim.size(),pos4 - pos3 - strdelim.size());
break;
}//end while (readFile.peek()!=EOF)
}// end
}
void DayBillMonthCharge::getPatternFilesMsg(string strfilename){
int ifileorder = 1;
file_item_vector.clear();
set < string > fileset;
Glob::glob(strfilename, fileset);
set<std::string>::iterator it = fileset.begin();
for (; it != fileset.end(); ++it) {
std::ifstream *readFile = new std::ifstream;
(*readFile).open((*it).c_str(), std::ifstream::in);
File_Item *file_item_case = new File_Item(ifileorder, *it, true,readFile);
file_item_vector.push_back(*file_item_case);
delete file_item_case;
ifileorder++;
}
}
int DayBillMonthCharge::main(const std::vector<std::string>& args) {
if (!_helpRequested) {
int sz = (int) _billing_day.size();
if (sz != 8) {
return Application::EXIT_USAGE;
}
if((_payment_type!=1)&&(_payment_type!=2)){
return Application::EXIT_USAGE;
}
_log_name = _corp_head+"DayBillMonthCharge";
_log_dir="$HOME/log/day_report/" ;
_log_dir = Poco::Path::expand(_log_dir);
File logDir(_log_dir);
logDir.createDirectories();
Poco::trimInPlace(_log_dir);
DayBillLog * daylog = new DayBillLog(_corp_head);
_filedir = "$HOME/data/day_report/daybill_monthcharge_"+ _billing_day.substr(0, 6)+"/data/"+_corp_head+"/";
_filedir = Poco::Path::expand(_filedir);
File datafileDir(_filedir);
datafileDir.createDirectories();
_configdir="$HOME/data/day_report/daybill_monthcharge_"+ _billing_day.substr(0, 6)+"/config/"+_corp_head+"/";
_configdir = Poco::Path::expand(_configdir);
File configDir(_configdir);
configDir.createDirectories();
Poco::trimInPlace(_filedir);
_logtext = "量收号码月累计程序开始";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
int billingday;
if(_billing_day.substr(6,1)=="0"){
billingday=atoi(_billing_day.substr(7,1).c_str());
}else{
billingday=atoi(_billing_day.substr(6,2).c_str());
}
LocalDateTime this_day(atoi((_billing_day.substr(0,4)).c_str()),atoi((_billing_day.substr(4,2)).c_str()),billingday, 00, 00,00);
Timespan span(-1*Timespan::DAYS);
LocalDateTime last_time= this_day + span;
char last_day[8];
sprintf(last_day,"%d%02d%02d",last_time.year(),last_time.month(),last_time.day());
_last_date=string(last_day);
LocalDateTime now;
LocalDateTime now_last_time= now + span;
sprintf(last_day,"%d%02d%02d",now_last_time.year(),now_last_time.month(),now_last_time.day());
_yesterday=string(last_day);
_result_db_table="IF_SURE_SRC_DAY_SERV_"+_billing_day.substr(0,6);
_result_yddb_table="TB_BILL_SUM_SERV_POSI_"+_billing_day.substr(0,6);
_result_yddbsz_table="TB_BILL_SUM_SERV_POSIT_"+_billing_day.substr(0,6);
if(_billing_day==_yesterday){
ForkMain(READBILLTABLE);
ForkMain(MERGEFILE);//汇总进程级文件
if(_payment_type==YFFPAYMENT){
mergeYffFinalResultProc();
}
if(_billing_day.substr(6,2)=="01"){
}else{
ForkMain(MERGEINCREMENT);
}
bool if_bill_success=false;
bool if_gy_success=false;
creatTable();
if(_payment_type==HFFPAYMENT){
if_bill_success=sqlloadDate(SQLLOADHFFBILL);
if_gy_success=sqlloadDate(SQLLOADHFFGY);
if(if_bill_success&&if_gy_success){
creatTableIndex();
insertDate();
}else{
_logtext = "插入号码资料表数据错误";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
}else if(_payment_type==YFFPAYMENT){
if_bill_success=sqlloadDate(SQLLOADYFF);
if(if_bill_success){
creatTableIndex();
insertDate();
}else{
_logtext = "插入号码资料表数据错误";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
}
}else{
string cmd;
if(_payment_type==HFFPAYMENT){
_logtext = "调用后付费存储过程SP_DAYBILL_NBRSUM开始";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
cmd = "$HOME/bin/SP_DAY_NBRSUM_HF.sh "+_corp_head;
system(cmd.c_str());
_logtext = "调用后付费存储过程SP_DAYBILL_NBRSUM结束";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}else if(_payment_type==YFFPAYMENT){
_logtext = "调用预付费存储过程SP_DAYBILL_NBRSUM开始";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
cmd = "$HOME/bin/yff/SP_DAY_NBRSUM_YF.sh";
system(cmd.c_str());
_logtext = "调用预付费存储过程SP_DAYBILL_NBRSUM结束";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
ForkMain(READSUMTABLE);
ForkMain(MERGEFILESPECIAL);
}
_logtext = "量收号码月累计程序结束";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
return 0;
}
int DayBillMonthCharge::main(const std::vector<std::string>& args) {
if (!_helpRequested) {
int sz = (int) _billing_day.size();
if (sz != 8) {
return Application::EXIT_USAGE;
}
if((_payment_type!=1)&&(_payment_type!=2)){
return Application::EXIT_USAGE;
}
_log_name = _corp_head+"DayBillMonthCharge";
_log_dir="$HOME/log/day_report/" ;
_log_dir = Poco::Path::expand(_log_dir);
File logDir(_log_dir);
logDir.createDirectories();
Poco::trimInPlace(_log_dir);
DayBillLog * daylog = new DayBillLog(_corp_head);
_filedir = "$HOME/data/day_report/daybill_monthcharge_"+ _billing_day.substr(0, 6)+"/data/"+_corp_head+"/";
_filedir = Poco::Path::expand(_filedir);
File datafileDir(_filedir);
datafileDir.createDirectories();
_configdir="$HOME/data/day_report/daybill_monthcharge_"+ _billing_day.substr(0, 6)+"/config/"+_corp_head+"/";
_configdir = Poco::Path::expand(_configdir);
File configDir(_configdir);
configDir.createDirectories();
Poco::trimInPlace(_filedir);
_logtext = "量收号码月累计程序开始";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
int billingday;
if(_billing_day.substr(6,1)=="0"){
billingday=atoi(_billing_day.substr(7,1).c_str());
}else{
billingday=atoi(_billing_day.substr(6,2).c_str());
}
LocalDateTime this_day(atoi((_billing_day.substr(0,4)).c_str()),atoi((_billing_day.substr(4,2)).c_str()),billingday, 00, 00,00);
Timespan span(-1*Timespan::DAYS);
LocalDateTime last_time= this_day + span;
char last_day[8];
sprintf(last_day,"%d%02d%02d",last_time.year(),last_time.month(),last_time.day());
_last_date=string(last_day);
LocalDateTime now;
LocalDateTime now_last_time= now + span;
sprintf(last_day,"%d%02d%02d",now_last_time.year(),now_last_time.month(),now_last_time.day());
_yesterday=string(last_day);
_result_db_table="IF_SURE_SRC_DAY_SERV_"+_billing_day.substr(0,6);
_result_yddb_table="TB_BILL_SUM_SERV_POSI_"+_billing_day.substr(0,6);
_result_yddbsz_table="TB_BILL_SUM_SERV_POSIT_"+_billing_day.substr(0,6);
if(_billing_day==_yesterday){
ForkMain(READBILLTABLE);
ForkMain(MERGEFILE);//汇总进程级文件
if(_payment_type==YFFPAYMENT){
mergeYffFinalResultProc();
}
if(_billing_day.substr(6,2)=="01"){
}else{
ForkMain(MERGEINCREMENT);
}
bool if_bill_success=false;
bool if_gy_success=false;
creatTable();
if(_payment_type==HFFPAYMENT){
if_bill_success=sqlloadDate(SQLLOADHFFBILL);
if_gy_success=sqlloadDate(SQLLOADHFFGY);
if(if_bill_success&&if_gy_success){
creatTableIndex();
insertDate();
}else{
_logtext = "插入号码资料表数据错误";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
}else if(_payment_type==YFFPAYMENT){
if_bill_success=sqlloadDate(SQLLOADYFF);
if(if_bill_success){
creatTableIndex();
insertDate();
}else{
_logtext = "插入号码资料表数据错误";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
}
}else{
string cmd;
if(_payment_type==HFFPAYMENT){
_logtext = "调用后付费存储过程SP_DAYBILL_NBRSUM开始";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
cmd = "$HOME/bin/SP_DAY_NBRSUM_HF.sh "+_corp_head;
system(cmd.c_str());
_logtext = "调用后付费存储过程SP_DAYBILL_NBRSUM结束";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}else if(_payment_type==YFFPAYMENT){
_logtext = "调用预付费存储过程SP_DAYBILL_NBRSUM开始";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
cmd = "$HOME/bin/yff/SP_DAY_NBRSUM_YF.sh";
system(cmd.c_str());
_logtext = "调用预付费存储过程SP_DAYBILL_NBRSUM结束";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
ForkMain(READSUMTABLE);
ForkMain(MERGEFILESPECIAL);
}
_logtext = "量收号码月累计程序结束";
daylog->printlog(_billing_day, _log_dir, _log_name, _logtext);
}
return 0;
}
2097
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
