c++:缓存重发

最近在写边缘端的数据缓存重发功能, 当边缘端网络情况恶劣,数据时常有发布出去的情况, 选择缓存重发是一个比较好的选择, 因此花了两天的时间写了一个缓存重发功能, 源码见下方链接, 分享一下, 不足之处望指出.
依赖:boost, 使用前需安装

//
// Created by z on 2021/7/5.
//

#ifndef BOOSTFILEPROCESS_MsgFlow_H
#define BOOSTFILEPROCESS_MsgFlow_H


#include "boost/filesystem.hpp"

#include <string>
#include <queue>
#include <mutex>
#include <thread>
#include <unordered_map>
#include <cstdio>
#include <sstream>
#include <iostream>
#include <fstream>
#include <time.h>
#include <sys/timeb.h>

#define  SEPARATION "#&&#"

class MsgFlow {
public:
    MsgFlow(std::function<bool(std::string)> func){
        loop_times = 5;
        sys_quit = false;
        count_successfully = 0;
        count_failed = 0;
        sys_data_path="./";

        LoadFileNames(sys_data_path);
        callfunc = func;

    };
    ~MsgFlow(){
        dump();
    }

    void SendToFlow(const std::string& eventStr);

    bool SendMessage(const std::string& data);

    void MoveData();

    void Split(const std::string& data);

    void LoadFileNames(const std::string& path);

    void SaveDatatoFile();

    void run();

    void dump();

    void print();
private:
    bool success_flag;
    int loop_times;
    std::string sys_data_path;
    std::queue<std::string> m_queue; //data queue
    std::queue<std::string> m_fail_queue; //failed queue
    std::mutex m_qmutex;   // data queue mutex
    std::mutex m_fqmutex;  //failed queue mutex;
    std::vector<std::pair<std::string, long int>> files;

    std::function<bool(std::string)> callfunc;


    int count_successfully;
    int count_failed;

public:
    bool sys_quit;
};
#endif //BOOSTFILEPROCESS_MsgFlow_H

构造函数,传入用户定义的发送函数, FlowRpc模块回调, 解耦.

#include "MsgFlow.h"

void MsgFlow::SendToFlow(const std::string& eventStr){
    std::unique_lock<std::mutex> qlock(m_qmutex, std::defer_lock);

    qlock.lock();
    {
        if (eventStr.empty()){
            qlock.unlock();
            return;
        }

        m_queue.push(eventStr);
        std::cout<<"push "<<m_queue.size()<<std::endl;
    }
    qlock.unlock();
}

void MsgFlow::run() {
    //新开一个线程, 来检查m_queue的状态
    //m_queue 为空 就向里面塞数据
    //fail_queue 数据过多, 存入磁盘
    //注意内存对齐  暂时未做
    std::thread t1([&](){
        MoveData();
    });
    t1.detach();
    std::unique_lock<std::mutex> m_qlock(m_qmutex, std::defer_lock);
    std::unique_lock<std::mutex> m_fqlock(m_fqmutex, std::defer_lock);

    //计算发送失败的次数
    int count = 0;
    int cnt = 0;
    while(true){
        std::this_thread::sleep_for(std::chrono::milliseconds(1));
        if (sys_quit) {
            dump();
            return;
        }
        //不断取数据
        if (m_queue.empty()){
            std::this_thread::sleep_for(std::chrono::milliseconds(1));
            continue;
        }
        m_qlock.lock();
        auto info = m_queue.front();
        m_queue.pop();
        m_qlock.unlock();


        while (cnt++ < loop_times ){
            //不断发送数据
            if (SendMessage(info)){
                cnt = 0;
                success_flag=true;
                std::cout<<"send successfully "<<count++<<std::endl;
                break;
            }
            std::this_thread::sleep_for(std::chrono::milliseconds(1));
        }
        

        if (cnt >= loop_times){
            success_flag=false;
            m_fqlock.lock();
            m_fail_queue.push(info);
            cnt = 0;
            m_fqlock.unlock();
        }

        
        if (m_fail_queue.size() >= 5){
            if (files.size()> 100)
            {
                auto top = files.front();
                files.erase(files.begin());
                //删除文件
                if(::remove(top.first.c_str())!=0){
                    std::cout<<"delete file failed!"<<std::endl;
                }
            }

            long int time = std::chrono::system_clock::now().time_since_epoch() / std::chrono::seconds(1);
            std::string name = sys_data_path+std::to_string(time)+".data.over";
            
            files.push_back(std::pair<std::string, long int>(name, time));
            std::ofstream ofs(name, std::ios::app);
            m_fqlock.lock();
            while(m_fail_queue.size() > 0){
                ofs<<m_fail_queue.front()<<SEPARATION;
                m_fail_queue.pop();
            }
            ofs.close();
            std::cout<<"m_faile_queue size is greater than 5!!! save in file and m_faile_queue size is: "<<m_fail_queue.size()<<name;
            m_fqlock.unlock();
        }
        
    }
}
void MsgFlow::SaveDatatoFile() {
    std::unique_lock<std::mutex> m_fqlock(m_fqmutex);
    if (m_fail_queue.size() >= 5){
        if (files.size()> 100)
        {
            auto top = files.front();
            files.erase(files.begin());
            //删除文件
            if(::remove(top.first.c_str())!=0){
                std::cout<<"delete file failed!"<<std::endl;
            }
        }

        long int time = std::chrono::system_clock::now().time_since_epoch() / std::chrono::seconds(1);
        std::string name = sys_data_path+std::to_string(time)+".data.over";

        files.push_back(std::pair<std::string, long int>(name, time));
        std::ofstream ofs(name, std::ios::app);
        while(m_fail_queue.size() > 0){
            ofs<<m_fail_queue.front()<<SEPARATION;
            m_fail_queue.pop();
        }
        ofs.close();
        std::cout<<"m_faile_queue size is greater than 5!!! save in file and m_faile_queue size is: "<<m_fail_queue.size()<<name<<std::endl;
    }
    m_fqlock.unlock();
}
bool MsgFlow::SendMessage(const std::string& data){
    //使用回调函数
    if (callfunc(data)){
        return true;
    } else
        return false;

}
void MsgFlow::Split(const std::string& data){
    int index = 0;
    int dataidx = 0;
    while ((dataidx = data.find(SEPARATION, index)) <= data.length())
    {
        std::string str = data.substr(index, dataidx);
        m_queue.push(str);
        index = dataidx+strlen(SEPARATION);
    }
    if (data.find_last_of(SEPARATION) != (data.length()-1))
        std::cout<<"not end of "<<SEPARATION<<", the lastest record maybe wrong";

}

void MsgFlow::MoveData() {
    std::unique_lock<std::mutex> m_qlock(m_qmutex, std::defer_lock);
    std::unique_lock<std::mutex> m_fqlock(m_fqmutex, std::defer_lock);
    while (true){
        if (sys_quit)
        {
            return;
        }
        if (m_queue.empty() && !success_flag)
            std::this_thread::sleep_for(std::chrono::milliseconds(10));

        if (m_queue.empty()){
            if (!m_fail_queue.empty() && success_flag){
                m_fqlock.lock();
                std::swap(m_fail_queue, m_queue);
                m_fqlock.unlock();
            }
        }

        if (!m_fail_queue.empty() && success_flag){
            //failed queue pop;
            std::cout <<"data from failed queue lock";
            m_fqlock.lock();
            auto info = m_fail_queue.front();
            m_fail_queue.pop();
            m_fqlock.unlock();
            std::cout<<"data from failed queue ulock";
            //data queue enqueue;
            if (!info.empty()){
                std::cout<<"failed queue to data queue lock";
                m_qlock.lock();
                m_queue.push(info);
                m_qlock.unlock();
                std::cout<<"failed queue to data queue unlock";
            }
                
        }
        if (m_fail_queue.empty() && m_queue.empty() && success_flag){


            //读本地文件 files 按时间升序 排列 ,第一个即历史最久的文件
            if (files.size() > 0){

                auto top = files.front();
                std::ifstream in(top.first);
                std::ostringstream oss;
                oss<< in.rdbuf();
                std::string str = oss.str();
                std::cout<<str<<std::endl;
                //将str分开存入m_queuue
                m_qlock.lock();
                Split(str);
                m_qlock.unlock();
                //从维护的vector 中删除记录
                files.erase(files.begin());
                std::cout<<files.begin()->first<<std::endl;
                for (auto fileit = files.begin(); fileit != files.end(); fileit++)
                    std::cout<<fileit->first<<std::endl;

                //删除文件
                if(::remove(top.first.c_str())!=0){
                    std::cout<<"delete file failed!"<<std::endl;
                }
            }
            success_flag = false;
        }
    }
}


void MsgFlow::LoadFileNames(const std::string& path){
    boost::filesystem::path p (path);
    try{
        if (exists(p)){
            if (is_regular_file(p))
                return ;

            if (is_directory(p)){
                for (boost::filesystem::directory_entry& x : boost::filesystem::directory_iterator(p)){
                    if (is_regular_file(x.path()) && x.path().string().find(".data.over") != std::string::npos){
                        std::cout << "   " << x.path() << '\n';
                        long int t =  static_cast<long int> (boost::filesystem::last_write_time(x));
                        std::cout << "time:" << t <<std::endl;
                        files.push_back(std::pair<std::string, long int>(x.path().string(),t));
                    }
                }
            }
            else
                std::cout <<"empty directory\n";
        }
        else
            std::cout << p << " does not exist\n";

        // 将文件按时间排序
        std::sort(files.begin(), files.end(),
                  [&](std::pair<std::string, long int> a, std::pair<std::string, long int> b){
                      return a.second < b.second;});

        return ;
    }
    catch (const boost::filesystem::filesystem_error& ex)
    {
        std::cout << ex.what() << '\n';
    }
}
void MsgFlow::dump(){
    //if this processor exit
    // dump the queue's content
    long int time = std::chrono::system_clock::now().time_since_epoch() / std::chrono::seconds(1);
    std::string name = std::to_string(time)+".data.over";
    std::ofstream ofs(name, std::ios::app);

    if (!m_queue.empty()){
        std::cout<<"m_queue not empty!"<<std::endl;
        while(m_queue.size()> 0){
            ofs<<m_queue.front()<<SEPARATION;
            m_queue.pop();
        }
    }

    // dump the failed queue's content
    if (!m_fail_queue.empty()){
        std::cout<<"m_fail_queue not empty"<<std::endl;
        while(m_fail_queue.size() > 0){
            ofs<<m_fail_queue.front()<<SEPARATION;
            m_fail_queue.pop();
        }
    }
    ofs.close();
}
void MsgFlow::print() {
    auto p_queue = m_queue;
    while(p_queue.size() > 0)
    {
        auto it = p_queue.front();
        p_queue.pop();
        std::cout<<it<<std::endl;
    }
}

其中run()函数负责

1. 发送数据: 从m_queue中取数据, 发送; 若发送失败超5次, 保存到m_fail_queue
2. 检查m_faile_queue.size()是否超过阈值, 超过则按规则写入文件

MoveData()函数

1. 若m_queue 为空, m_fail_queue不为空, 交换两个队列
2. 在1的情况下, 若m_queue 不为空, m_fail_queue 不为空, 可以从m_fail_queue 转存一个数据到m_queue
3. 在1, 2的情况下, 若m_queue和m_fail_queue都为空, 且数据文件列表不为空, 从文件读取数据, 用分隔符 SEPARATION 分隔开

dump() 函数

1. 在程序退出的时候, 检查数据队列, 若不为空, 写到文件

//main.cpp
#include "MsgFlow.h"
#include <iostream>
#include <zconf.h>
#include <fstream>
bool SendMethod(const std::string& data){
    //TODO 此处定义网络发送函数
    //此处暂时定义模拟发送函数, 模拟成功失败的概率
    struct timeb timeSeed;
    std::cout<<"in SendMethod"<<std::endl;
    ftime(&timeSeed);
    srand(timeSeed.time * 1000 + timeSeed.millitm);  // milli time
    int r = rand() %11;
    if (r % 7 == 0)
        return true;

    return false;
}
int main() {
    std::string data ="{\"type\":0,\"tbname\":\"trajectory\",\"dbname\":\"lan\",\"data\":{\"x\":\"105\",\"timestamp\":1625122637500,\"sn\":\"2020\",\"id\":\"298\",\"y\":\"408\",\"url\":\"001.jpg\"}}";

    auto func = std::bind(&SendMethod, std::placeholders::_1);
    MsgFlow frpc(func);


    std::thread t([&](){
        for (int idx = 0; idx < 100; ++idx){
            frpc.SendToFlow(data);
            // std::this_thread::sleep_for(std::chrono::milliseconds(10));
        }
    });

    std::thread t1([&](){
        frpc.run();
    });
    t.detach();
    t1.detach();

    // while(true)
        sleep(10);
    // sleep(500);
    frpc.sys_quit = true;
    sleep(10);


    return 0;
}

main函数用来模拟测试, 经测试, 正确

回顾:
可优化的方向

1. 使用线程条件变量来更改MoveData的运行机制, 减少cpu空转
2. 如果网络环境太差, 且不再生产数据, m_fail_queue.size() 大于0, 小于阈值, 会一直发不出去, 数据就会在m_fail_queue和m_queue里面循环.如果生产者持续生产数据,则没有问题.
3. 写数据的地方, 可以考虑内存对齐, 和数据压缩存储, 减小磁盘数据量
4. boost 依赖库的问题, 可以考虑直接使用std标准库, 减少依赖

具体代码见github