日志备份

#include <iostream>
#include <fstream>
#include <ctime>
#include <cstring>
#include <vector>
#include <deque>
//#include <queue>
#include <thread>
#include <algorithm>
#include <unordered_set>
#include <unordered_map>
#include"boost/algorithm/string.hpp"
using namespace std;
struct video
{
    
    string id;
    float hot;
    int num;
    int file_name;
    video(){}
    video(string i,float j,int k,int n):id(i),hot(j),num(k),file_name(n){}
    friend bool operator < (const video &v1,const video &v2)
    {
        if(v1.hot>v2.hot)return true;
        else if(v1.hot==v2.hot&&v1.id>v2.id) return true;
        return false;
    }
};
deque<video> output_que;                //输出队列
bool b_newtxt=false;
bool b_buffer=false;
bool b_arranging=false;
unordered_set<string> s_transcoded;     //记录已被转化的视频id
int txt_name=0;
int test_number=0;
double test_time=0;
fstream outfile_log;
fstream outfile_transcoded;
string log_txt;
string transcoded_id_txt;
deque<string> txt_buffer_que;//输出线程t_output将修改日志写入到缓存中，由t_read_newtxt定期写入硬盘
int parser_txt(string sname,vector<video> &vi)
{
    string s;
    vector<string> vs;
    ifstream infile_txt(sname,ios::in);
    if(!infile_txt){return -1;}
    while(!infile_txt.eof())
    {
        vector<string>().swap(vs);
        getline(infile_txt, s);
        if(s.size()==0) break;
        boost::split(vs,s,boost::is_any_of(" "),boost::token_compress_on);
        s=vs[0]+"-"+vs[1];
        if(s_transcoded.count(s)==0)
        {
            video v(s,stof(vs[2]),1,txt_name);
            vi.push_back(v);
        }
        
    }
    infile_txt.close();
    return 0;
}
void read_file(vector<video> &vi)
{
    string sname=to_string(txt_name)+".txt";
    if(parser_txt(sname,vi)==-1)
    {
        for(int i=1;;i++)
        {
            sname=to_string(txt_name)+"-"+to_string(i)+".txt";
            if(parser_txt(sname,vi)==-1)
            {
                //if(i==1) cout<<"openfile  :"<<sname<<" fail"<<endl;
                break;
            }
        }
    }
    
}
void  normal_initialize()
{
    vector<video> vi;
    outfile_log.open(log_txt,ios::in|ios::out|ios::trunc);
    outfile_transcoded.open(transcoded_id_txt,ios::out|ios::trunc);
    read_file(vi);
    sort(vi.begin(),vi.end());
    for(video i:vi)
    {
        output_que.push_back(i);
        //outfile_log<<"add id :"<<i.id<<","<<i.hot<<","<<i.num<<","<<i.file_name<<endl;
        string temp_log="add "+i.id+":"+to_string(i.hot)+","+to_string(i.num)+","+to_string(i.file_name);
        txt_buffer_que.push_back(temp_log);
    }
        
    txt_name++;
}
void settimer_output(deque<video> &que,float &output_speed,ofstream &outfile_txt)//实现定时器
{
    clock_t start, finish;
    start = clock();
    double totaltime = 0;
    while (1)
    {
        finish = clock();
        totaltime = (double)(finish - start)/CLOCKS_PER_SEC;
        if (totaltime > output_speed)
        {
            if(!que.empty())
            {
                outfile_txt<<que.front().id<<endl;
                outfile_transcoded<<que.front().id<<endl;
                //outfile_log<<"delete id: "<<que.front().id<<endl;
                string temp_log="delete "+que.front().id;
                txt_buffer_que.push_back(temp_log);
                test_time+=totaltime;
                cout<<test_number++<<"  "<<que.front().id<< "  "<<que.front().hot<<" "<<test_time<<" "<<que.front().file_name<<endl;
                s_transcoded.insert(que.front().id);
                //cout<<temp_log<<endl;
                que.pop_front();
            }
            return;
        }
    }
}
void output_regular(int index,deque<video> &que,float &output_speed, ofstream &outfile_txt)
{
    while(1)
    {
        settimer_output(que,output_speed,outfile_txt);
        if(que.empty()) {break;}
        if(index==1&&b_newtxt){break;}//index=1,表示正在从output_queue中输出id，当读取到新文件时应停止输出
    }
}
void output_regular_from_buffer(int &buffer_size,float &output_speed,ofstream &outfile_txt)
{
    deque<video> buffer_que;
    for(int i=0;i<buffer_size&&i<output_que.size();i++)
    {
        buffer_que.push_back(output_que.front());
        s_transcoded.insert(output_que.front().id);
        output_que.pop_front();
    }
    b_buffer=true;
    output_regular(2,buffer_que,output_speed,outfile_txt);
}
void output(float output_speed,int buffer_size,string out_txt,string log_txt)
{
    ofstream outfile_txt(out_txt,ios::out|ios::trunc);
    while(1)
    {
        output_regular(1,output_que,output_speed,outfile_txt);//从output_queue中输出待转码视频id
        output_regular_from_buffer(buffer_size,output_speed,outfile_txt);//从缓冲区buffer_que中输出待转码视频id
        while(b_newtxt) { int a=1;} //当b_newtxt==true，说明新旧队列合并没有完成，应进入等待。
    }
    outfile_txt.close();
}
vector<string> arrange_log(int t)
{
    outfile_log.close();
    outfile_log.open(log_txt,ios::in);
    vector<string> vs;
    unordered_map<string,string> add_map;
    unordered_map<string,string> alter_map;
    string s;
    while(!outfile_log.eof())
    {
        vector<string>().swap(vs);
        getline(outfile_log, s);
        if(s.size()==0) break;
        boost::split(vs,s,boost::is_any_of(" :"),boost::token_compress_on);
        if(vs[0]=="add")
        {
            add_map[vs[1]]=vs[2];
        }
        else if(vs[0]=="alter")
        {
            add_map.erase(vs[1]);
            alter_map[vs[1]]=vs[2];
        }
        else
        {
            add_map.erase(vs[1]);
            alter_map.erase(vs[1]);
        }
    }
    outfile_log.close();
    outfile_log.open(log_txt,ios::out|ios::trunc);
    vector<string>().swap(vs);
    for(auto i:add_map)
    {
        outfile_log<<"add "<<i.first<<":"<<i.second<<endl;
    }
    for(auto i:alter_map)
    {
        outfile_log<<"alter "<<i.first<<":"<<i.second<<endl;
        
    }
    if(t==1)
    {
        for(auto i:add_map)
        {
            s=i.first+","+i.second;
            vs.push_back(s);
        }
        for(auto i:alter_map)
        {
            s=i.first+","+i.second;
            vs.push_back(s);
        }
    }
    return vs;
}
void settimer_read_newtxt(float &msec,vector<video> &vi,float &write_log_speed,float &arrange_time)//实现定时器
{
    clock_t start, finish;
    start = clock();
    double totaltime = 0;
    while (1)
    {
        finish = clock();
        totaltime = (double)(finish - start)/CLOCKS_PER_SEC;

        if(!txt_buffer_que.empty())
        {
            outfile_log<<txt_buffer_que.front()<<endl;      //将缓冲区里的日志持久化到硬盘文件中
            txt_buffer_que.pop_front();
        }
        if(totaltime>arrange_time)
        {
            arrange_log(0);                                //整理日志文件
            arrange_time+=arrange_time;
        }
        totaltime = (double)(finish - start)/CLOCKS_PER_SEC;
        if (totaltime > msec)
        {
            //while(vi.size()==0)
            
            read_file(vi);       //如果到了一个小时没有发现新文件，应进入循环，直到读取到文件
            if(vi.size()!=0)
            {
                b_newtxt=true;            //使输出线程将输出源切换为缓冲区
                txt_name++;
                return;
                
            }
        }
    }
}
void merge_data(vector<video> &vi,int &output_que_size)
{
   // queue<video> output_que;
    unordered_map<string,video> m_new;
    unordered_map<string,video> m_output;
    unordered_map<string,video> alter_map;
    unordered_map<string,video> add_map;
    for(video i:output_que)
    {
        m_output[i.id]=i;
    }
    for(video i:vi)
    {
        if(s_transcoded.count(i.id)==0)
        {
            m_new[i.id]=i;
            if(m_output.count(i.id)==0)
            {
                add_map[i.id]=i;
            }
        }
    }
    vector<video>().swap(vi);
    while(!output_que.empty())
    {
        video temp=output_que.front();
        float num_index;
        output_que.pop_front();
        if(m_new.count(temp.id)==0)
        {
            num_index=1+(temp.num-1)*0.15;
            temp.hot=temp.hot*0.5*num_index;
        }
        else
        {
            temp.num++;
            num_index=1+(temp.num-1)*0.15;
            temp.hot=(m_new[temp.id].hot*0.7+temp.hot*0.3)*num_index;
            temp.file_name=m_new[temp.id].file_name;
            alter_map[temp.id]=temp;
            
        }
        m_new[temp.id]=temp;
    }
    for(auto i:m_new)
    {
        vi.push_back(i.second);
    }
    sort(vi.begin(),vi.end());
    for(video i:vi)
    {
        output_que.push_back(i);
    }
    m_new.clear();
    m_output.clear();
    while(output_que.size()>=output_que_size)
    {
        video temp=output_que.back();
        output_que.pop_back();
        if(add_map.count(temp.id)!=0)
        {
            add_map.erase(temp.id);
        }
        else if(alter_map.count(temp.id)!=0)
        {
            alter_map.erase(temp.id);
            //outfile_log<<"delete id: "<<temp.id<<endl;
            string temp_log="delete "+temp.id;
            txt_buffer_que.push_back(temp_log);
        }
        else
        {
            //outfile_log<<"delete id: "<<temp.id<<endl;
            string temp_log="delete "+temp.id;
            txt_buffer_que.push_back(temp_log);
        }
    }
    for(auto i:add_map)
    {
        //outfile_log<<"add id :"<<i.second.id<<","<<i.second.hot<<","<<i.second.num<<","<<i.second.file_name<<endl;
        string temp_log="add "+i.second.id+":"+to_string(i.second.hot)+","+to_string(i.second.num)+","+to_string(i.second.file_name);
        txt_buffer_que.push_back(temp_log);
    }
    for(auto i:alter_map)
    {
        //outfile_log<<"alter id :"<<i.second.id<<","<<i.second.hot<<","<<i.second.num<<","<<i.second.file_name<<endl;
        string temp_log="alter "+i.second.id+":"+to_string(i.second.hot)+","+to_string(i.second.num)+","+to_string(i.second.file_name);
        txt_buffer_que.push_back(temp_log);
    }
   // b_buffer=false;
    //b_newtxt=false;
}
void read_newtxt(float read_time,int output_que_size,float write_log_speed,float arrange_time)
{
    vector<video> vi;
    while(1)
    {
        vector<video>().swap(vi);
        settimer_read_newtxt(read_time,vi,write_log_speed,arrange_time);
        while(!b_buffer) { int a=1;}         //当b_buffer==false，说明缓冲区还没建立完毕，应进入等待。
        merge_data(vi,output_que_size);                     //将output_queue与vi中的数据合并放入output_queue中
        b_buffer=false;
        b_newtxt=false;
        
    }
    
}
void data_recover()
{
    ifstream infile_transcoded(transcoded_id_txt,ios::in);
    if(!infile_transcoded){return ;}
    string s;
    while(!infile_transcoded.eof())
    {
        getline(infile_transcoded, s);
        s_transcoded.insert(s);
    }
    infile_transcoded.close();
    vector<string> v_log=arrange_log(1);//在还原前先把日志整理一下
    vector<string> vs;
    vector<video> v_video;
    outfile_log.close();
    int now_time=0;
    for(string s:v_log)
    {
        vector<string>().swap(vs);
        boost::split(vs,s,boost::is_any_of(", "),boost::token_compress_on);
        video temp(vs[0],stof(vs[1]),stoi(vs[2]),stoi(vs[3]));
        now_time=max(now_time,temp.file_name);
        v_video.push_back(temp);
    }
    for(video i:v_video)                        //获取最新的时间，并对以前的视频进行hot计算
    {
        if(i.file_name<now_time)
        {
            float num_index=1+(i.num-1)*0.15;
            num_index*=0.5;
            i.hot=i.hot*pow(num_index,now_time-i.file_name);
        }
    }
    sort(v_video.begin(),v_video.end());
    for(video i:v_video)
    {
        output_que.push_back(i);
    }

    outfile_log.open(log_txt,ios::in|ios::out|ios::app);
    outfile_transcoded.open(transcoded_id_txt,ios::out|ios::app);
    
}
                                                //txt文件命名示例 11-1.txt，11-2.txt,12.txt
int main(int argc, char *argv[])
{
    txt_name=11;                                 //初始化时使用的文件名11.txt
    string out_txt="output.txt";                 //接收输出的文件名
    log_txt="data_log";                     //保存操作日志的文件名
    transcoded_id_txt="transcoded_id.txt";     //保存已转码视频id的文件名
    float output_speed=0.5;                          //输出速度为每output_speed秒一个
    int output_que_size=200;                         //输出队列的最大容量
    int buffer_size=5;                           //缓冲区容量
    float read_time=10;                            //每time秒读取一次新txt
    float write_log_speed=output_speed;           //输出线程t_output将修改日志写入到缓存中txt_buffer_que，由t_read_newtxt定期写入硬盘
    float arrange_time=5;                         //每arrange_time对硬盘中的日志文件data_log进行整理重写，减少文件的体积
    normal_initialize();                           //正常由txt_name.txt初始化
    //data_recover();                                 //服务器重启后，从硬盘还原数据
    thread t_output(output,output_speed,buffer_size,out_txt,log_txt);   //输出线程
    thread t_read_newtxt(read_newtxt,read_time,output_que_size,write_log_speed,arrange_time);                //读取新文件线程
 
    t_output.join();
    t_read_newtxt.join();
    return 0;
}