server.cpp
#include "httplib.h"
#include <boost/filesystem.hpp>
#include <iostream>
#include <sstream>
#include <fstream>
#include "compress.hpp"
#define SERVER_BASE_DIR "www"
#define SERVER_ADDR "0.0.0.0"
#define SERVER_PORT 9000
#define SERVER_BACKUP_DIR SERVER_BASE_DIR"/list/"
using namespace httplib;
namespace bf = boost::filesystem;
CompressStore cstor;
class CloudServer{
private:
SSLServer srv;
public:
CloudServer(const char* cert, const char* key): srv(cert, key){
bf::path base_path(SERVER_BASE_DIR);
//如果文件不存在
if (!bf::exists(base_path)){
//创建目录
bf::create_directory(base_path);
}
bf::path list_path(SERVER_BACKUP_DIR);
if (!bf::exists(list_path)){
bf::create_directory(list_path);
}
}
bool Start(){
srv.set_base_dir(SERVER_BASE_DIR);
srv.Get("/(list(/){0,1}){0,1}", GetFileList);
srv.Get("/list/(.*)", GetFileData);
srv.Put("/list/(.*)", PutFileData);
srv.listen("SERVER_ADDR", SERVER_PORT);
return true;
}
private:
static void PutFileData(const Request &req, httplib::Response &rsp)
{
std::cout << "backup file " << req.path << "\n";
if (!req.has_header("Range")){
rsp.status = 400;
return;
}
std::string range = req.get_header_value("Range");
int64_t range_start;
if (RangeParse(range, range_start) == false){
rsp.status = 400;
return;
}
std::string real = SERVER_BASE_DIR + req.path;
cstor.SetFileData(real, req.body, range_start);
std::ofstream file(real, std::ios::binary | std::ios::trunc);
if (!file.is_open()){
std::cerr << "open file " << real << "error\n";
rsp.status = 500;
return;
}
file.seekp(range_start, std::ios::beg);
file.write(&req.body[0], req.body.size());
if (!file.good()){
std::cerr << "file write body error\n";
rsp.status = 500;
return;
}
file.close();
return;
}
static bool RangeParse(std::string &range, int64_t &start){
//Range: bytes=start-end
size_t pos1 = range.find("=");
size_t pos2 = range.find("=");
if (pos1 == std::string::npos || pos2 == std::string::npos){
std::cerr << "range:[" << range << "] format error\n";
return false;
}
std::stringstream rs;
rs << range.substr(pos1 + 1, pos2 - pos1 - 1);
rs >> start;
return true;
}
//文件列表信息获取,定义成静态就是避免了this指针
static void GetFileList(const Request &req, httplib::Response &rsp){
std::vector<std::string> list;
cstor.GetFileList(list);
std::string body;
body = "<html><body><ol><hr />";
for (auto i:list){
bf::path path(i);
std::string file = path.filename().string();
std::string uri = "/list/" + file;
body += "<h4><li>";
body += "<a href='";
body += uri;
body += "'>";
body += file;
body += "</a>";
body += "</li></h4>";
//std::string file = item_begin->path().string();
//std::cerr << "file:" << file << std::endl;
}
body += "<hr /></ol></body></html>";
rsp.set_content(&body[0], "text/html");
return;
}
//获取文件数据(文件下载)
static void GetFileData(const Request &req, httplib::Response &rsp){
std::string real = SERVER_BASE_DIR + req.path;
std::string body;
cstor.GetFileData(real, body);
//正文只能给一次,文件不能太大
rsp.set_content(body, "text/plain"); //plain文件下载
}
};
void thr_start(){
cstor.LowHeatFileStore();
}
int main()
{
std::thread thr(thr_start);
thr.detach();
CloudServer srv("./cert.pem","./key.pem");
srv.Start();
return 0;
}
CloudClient.hpp
#define _D_SCL_SECURE_NO_WARNINGS 1
#ifndef __M_CLOUD_H__
#define __M_CLOUD_H__
#include <fstream>
#include <sstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string.hpp>
#include <thread>
#include "httplib.h"
#define SERVER_IP "192.168.122.132"
#define SERVER_PORT 9000
#define CLIENT_BACKUP_DIR ".\\backup"
#define CLIENT_BACKUP_INFO_FILE ".\\back.list"
#define RANGE_MAX_SIZE (10 << 20) //10MB
#define BACKUP_URI "/list/"
namespace bf = boost::filesystem;
class ThrBackUp
//封装一个类,使用一个参数来传递
{
private:
std::string _file;
int64_t _range_start;
int64_t _range_len;
public:
bool _res;
public:
ThrBackUp(const std::string &file, int64_t start, int64_t len) :_res(true),
_file(file), _range_start(start), _range_len(len) {}
void Start(){
//获取文件的range分块数据
std::stringstream ss;
ss << "backup file" << _file << "range:[" << _range_start << "-" << _range_len << "\n";
std::cout << ss.str();
std::ifstream path(_file, std::ios::binary);
if (!path.is_open()){
std::cerr << "range backup file " << _file << "failed!\n";
_res = false;
return;
}
//跳转到range的起始位置
path.seekg(_range_start, std::ios::beg);//跳转
std::string body;
body.resize(_range_len);
//读取文件中range分块的文件数据
path.read(&body[0], _range_len);
if (!path.good()){
std::cerr << "read file" << _file << " range data failed\n";
_res = false;
return;
}
path.close();
//上传range数据
bf::path name(_file);
//组织上传的uri路径 method url version
//PUT /list/filename HTTP/1.1
std::string url = BACKUP_URI + name.filename().string();//native 不要双引号
//实例化一个httplib的客户端对象
httplib::Client cli(SERVER_IP, SERVER_PORT);
//定义HTTP请求头信息
httplib::Headers hdr;
hdr.insert(std::make_pair("Content-Length", std::to_string(_range_len)));
std::stringstream tmp;
tmp << "bytes=" << _range_start << "-" << _range_start + _range_len - 1;
hdr.insert(std::make_pair("Range", tmp.str().c_str()));
//通过实例化的Client向服务端发送PUT请求
auto rsp = cli.Put(url.c_str(), hdr, body, "text/plain");
if (rsp &&rsp->status != 200){
_res = false;
}
else {
std::stringstream ss;
ss << "backup file" << _file << "] range:[" << _range_start << "-" << _range_len << "] backup success\n";
std::cout << ss.str();
return;
}
}
};
class CloudClient
{
public:
CloudClient(){
bf::path file(CLIENT_BACKUP_DIR);
if (!bf::exists(file)){
bf::create_directory(file);
}
}
private:
std::unordered_map<std::string, std::string> _backup_list;
//CloudClient(){}
private:
//获取备份信息
bool GetBackupInfo()
{
//filename1 etag\n
//filename2 etag\n
//实例化path对象
bf::path path(CLIENT_BACKUP_INFO_FILE);
if (!bf::exists(path)){
std::cerr << "list file" << path.string() << "is not exists\n";
return false;
}
int64_t fsize = bf::file_size(path);
if (fsize == 0){
std::cerr << "have no backup info\n";
return false;
}
std::string body;
body.resize(fsize);
std::ifstream file(CLIENT_BACKUP_INFO_FILE, std::ios::binary);
if (!file.is_open()){
std::cerr << "list file open error\n";
return false;
}
file.read(&body[0], fsize);
if (!file.good()){
std::cerr << "read list file body error\n";
return false;
}
file.close();
std::vector<std::string> list;
boost::split(list, body, boost::is_any_of("\n"));//切割后放到list中
for (auto i : list){
//filename2 etag
size_t pos = i.find(" ");
if (pos == std::string::npos)
continue;
std::string key = i.substr(0, pos);
std::string val = i.substr(pos + 1);
_backup_list[key] = val;
}
return true;
}
//将备份信息保存在指定文件中
bool SetBackuoInfo()
{
std::string body;
for (auto i : _backup_list){
body += i.first + " " + i.second + "\n";
}
std::ofstream file(CLIENT_BACKUP_INFO_FILE, std::ios::binary);
if (!file.is_open()){
std::cerr << "open list file error\n";
return false;
}
file.write(&body[0], body.size());
if (!file.good()){
std::cerr << "set backuo info error\n";
return false;
}
file.close();
return true;
}
//目录监控
bool BackupDirListen(const std::string &path)
{
bf::directory_iterator item_begin(path);
bf::directory_iterator item_end;
for (; item_begin != item_end; ++item_begin){
// /list/abc/abc.txt bc.txt
if (bf::is_directory(item_begin->status())){
BackupDirListen(item_begin->path().string());
continue;
}
if (FileIsNeedBackup(item_begin->path().string()) == false){
continue;
}
std::cerr << "file:[" << item_begin->path().string() << "need backup\n";
if (PutFileData(item_begin->path().string()) == false){
continue;
}
//上传成功后
//记录文件Etag信息
AddBackInfo(item_begin->path().string());
}
return true;
}
//添加一个信息
bool AddBackInfo(const std::string &file)
{
//etag = "mtime-fsize"
std::string etag;
if (GetFileEtag(file, etag) == false)
return false;
_backup_list[file] = etag;
}
static void thr_start(ThrBackUp *backup_info)
{
backup_info->Start();
std::cout << "into thr_start\n";
return;
}
//备份文件
bool PutFileData(const std::string &file)
{
//按大小对文件内容进行分块传输(10MB)
//通过获取分块传输是否成功判断整个文件是否上传成功
//选择多线程处理
//1.获取文件大小
int64_t fsize = bf::file_size(file);
if (fsize <= 0){
std::cerr << "file " << file << "unnecessary backup\n";
return false;
}
//2.计算总共需要分多少块,得到每块大小以及起始位置
//3.循环创建线程,在线程中上传文件数据
int count = (int)fsize / RANGE_MAX_SIZE;
std::vector<ThrBackUp>thr_res;
std::vector<std::thread> thr_list;
std::cerr << "file:[" << file << "]fsize:[" << fsize << "] count:[" << count + 1 << "]\n";
// fsize = 500;
for (int i = 0; i <= count; ++i){
int64_t range_start = i * RANGE_MAX_SIZE; //起始位置
int64_t range_end = (i + 1) * RANGE_MAX_SIZE - 1;
if (i == count){
range_end = fsize - 1;
}
int64_t range_len = range_end - range_start + 1;
ThrBackUp backup_info(file, range_start, range_len);
std::cerr << "file:[" << file << "] range:[" << range_start << "-" << range_end << "]-" << range_len << "\n";
thr_res.push_back(backup_info);
thr_list.push_back(std::thread(thr_start, &thr_res[i]));
//std::thread thr(thr_start, file, range_start, range_len, &thr_res[i]);
}
/*for (int i = 0; i <= count; ++i)
{
thr_list.push_back(std::thread(thr_start, &thr_res[i]));
}*/
//4.等待所有线程退出,判断文件上传结果
bool ret = true;
for (int i = 0; i <= count; ++i){
thr_list[i].join();
if (thr_res[i]._res == true)
continue;
}
ret = false;
//5.上传成功,则添加文件的备份信息记录。 native() //不要双引号
if (ret == false){
return false;
}
//AddBackInfo(file);
std::cerr << "file:[" << file << "] backup success\n";
return true;
}
bool FileIsNeedBackup(const std::string &file)
{
std::string etag;
if (GetFileEtag(file, etag) == false)
return false;
auto it = _backup_list.find(file);
if (it != _backup_list.end() && it->second == etag){
//不需要备份
return false;
}
return true;
}
bool GetFileEtag(const std::string &file, std::string &etag)
{
bf::path path(file);
if (!bf::exists(path))
{
std::cerr << "get file" << file << "etag error\n";
return false;
}
int64_t fsize = bf::file_size(path);
int64_t mtime = bf::last_write_time(path);
std::stringstream tmp;
tmp << std::hex << fsize << "-" << std::hex << mtime;
etag = tmp.str();
return true;
}
public:
bool Start()
{
GetBackupInfo();//获取备份信息
while (1)
{
//对文件目录进行监控
BackupDirListen(CLIENT_BACKUP_DIR);
SetBackuoInfo();
Sleep(10);
}
return true;
}
};
#endif
CloudClient.cpp
#include "CloudClient.hpp"
int main()
{
CloudClient client;
client.Start();
return 0;
}
compress.hpp
#include <thread>
#include <boost/filesystem.hpp>
#include <fstream>
#include <boost/algorithm/string.hpp>
#include <sys/stat.h>
#include <zlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <pthread.h>
#include <sys/file.h>
#include <sstream>
#define UNGZIPFILE_PATH "www/list"
#define GZIPFILE_PATH "/www/zip"
#define RECORD_FILE "record.list"
#define HEAT_TIME 10
namespace bf = boost::filesystem;
class CompressStore{
private:
std::string _file_dir;
//用于保存文件列表
std::unordered_map<std::string, std::string> _file_list;
pthread_rwlock_t_rwlock;
private:
//1.每次压缩存储线程启动时候,从文件中读取列表信息
bool GetListRecord()
{
//filename, gzipfilename
bf::path name(RECORD_FILE);
if (!bf::exists(name)){
std::cerr << "record file is no exists\n";
return false;
}
std::ifstream file(RECORD_FILE, std::ios::binary);
if (!file.is_open()){
std::cerr << "open record file read error\n";
return false;
}
int64_t fsize = bf::file_size(name);
std::string body;
body.resize(fsize);
file.read(&body[0], fsize);
if (!file.good()){
std::cerr << "record file body read error\n";
return false;
}
file.close();
std::vector<std::string> list;
boost::split(list, body, boost::is_any_of("\n")); //切割
for (auto i : list){
//filename gzipname
size_t pos = i.find(" ");
if (pos == std::string::npos){
continue;
}
std::string key = i.substr(0, pos);
std::string val = i.substr(pos + 1);
_file_list[key] = val;
}
return true;
}
//2.每次压缩存储完毕,都将列表信息存储到文件中
bool SetListRecord(){
std::stringstream tmp;
for (auto i : _file_list){
tmp << i.first << " " << i.second << "\n";
}
std::ofstream file(RECORD_FILE, std::ios::binary| std::ios::trunc);
if (!file.is_open()){
std::cerr << "record file open error\n";
return false;
}
file.write(&tmp.str().c_str(), tmp.str().size());
if (!file.good()){
std::cerr << "record file write body error\n";
return false;
}
file.close();
}
2.1. 对文件进行压缩存储
//bool (){
// if (!bf::exists(UNGZIPFILE_PATH)){
// bf::create_directory(UNGZIPFILE_PATH);
// }
// bf::directory_iterator item_begin(UNGZIPFILE_PATH);
// bf::directory_iterator item_end;
// for (; item_begin != item_end; ++item_begin){
// if (bf::is_directory(item_begin->status)){
// continue;
// }
// std::string name = item_begin->path().string();
// if (!IsNeedCompress(name)){
// continue;
// }
// if (CompressFile(name)){
// std::cerr << "file:[" << name << "] store sucess\n";
// }
// else{
// std::cerr << "file:[" << name << "] store failed\n";
// }
// }
// return true;
//}
//2.2. 判断文件是否需要压缩
bool IsNeedCompress(std::string &file){
struct stat st;
if (stat(file.c_str(), &st) < 0){
std::cerr << "get file:[" << file << "] stat error\n";
return false;
}
time_t cur_time = time(NULL);
time_t acc_time = st.st_atime; //最后一次访问时间
if ((cur_time - acc_time) < HEAT_TIME){
return false;
}
return true;
}
//2.3. 对文件进行压缩存储
bool CompressFile(std::string &file,std::string &gzip{
int fd = open(file.c_str(), O_RDONLY);
if (fd < 0){
std::cerr << "com open file:[" << file << "] error\n";
return false;
}
//std::string gzfile = file + ".gz";
gzFile gf = gzopen(gzip.c_str(), "wb");
if (gf == NULL){
std::cerr << "com open gzip:[" << gzip << "] error\n";
return false;
}
int ret;
char buf[1024];
flock(fd, LOCK_SH);
while ((ret = read(fd, buf, 1024)) > 0){
gzWrite(gf, buf, ret);
}
flock(fd, LOCK_UN);
close(fd);
gzclose(gf);
unlink(file.c_str());
return true;
}
//3. 对文件进行解压缩
bool UnCompressFile(std::string &gzip,std::string &file{
int fd = open(file.c_str(), O_CREAT | O_WRONLY, 0664);
if (fd < 0){
std::cerr << "open file" << file << " failed\n";
return false;
}
gzFile gf = gzopen(gzip.c_str(), "rb");
if (gf == NULL){
std::cerr << "open gzip " << gzip << " failed\n";
close(fd);
return false;
}
int ret;
char buf[1024] = { 0 };
flock(fd, LOCK_EX);
while ((ret = gzread(gf, buf, 1024)) > 0){
int len = write(fd, buf, ret);
if (len < 0){
std::cerr << "get gzip data failed\n";
gzclose(gf);
close(fd);
flock(fd, LOCK_UN);
return false;
}
}
flock(fd, LOCK_UN);
gzclose(gf);
close(fd);
unlink(gzip.c_str());
return true;
}
bool GetNormalFile(std::string &name, std::string &body){
int64_t fsize = bf::file_size(name);
body.resize(fsize);
int fd = open(name.c_str(), O_RDONLY);
if (fd < 0){
std::cerr << "open file " << name << "failed\n";
return false;
}
flock(fd, LOCK_SH);
int ret = read(fd, &body[0], fsize);
flock(fd, LOCK_UN);
if (ret != fsize){
std::cerr << "get file " << name << "body error\n";
close(fd);
return false;
}
close(fd);
return true;
}
//目录检测,获取目录中的文件名
//1. 判断文件是否需要压缩存储
//2. 文件压缩存储
bool DirectoryCheck(){
if (bf::exists(UNGZIPFILE_PATH)){
bf::create_directory(UNGZIPFILE_PATH);
}
bf::directory_iterator item_begin(UNGZIPFILE_PATH);
bf::directory_iterator item_end;
for (; item_begin != item_end; ++item_begin){
if (bf::is_directory(item_begin->status())){
continue;
}
std::string name = item_begin->path().string();
std::string gzip = GZIPFILE_PATH+item_begin->path().filename().string() + ".gz";
if (IsNeedCompress(name)){
CompressFile(name, gzip);
AddFileRecord(name, gzip);
}
}
return true;
}
public:
CompressStore(){
pthread_rwlock_inti(&_rwlock,NULL);
if (!bf::exists(GZIPFILE_PATH)){
bf::create_directory(GZIPFILE_PATH);
}
}
~CompressStore(){
pthread_rwlock_destroy(&_rwlock);
}
//向外提供获取文件列表功能
bool GetFileList(std::vector<std::string> &list)
{
pthread_rwlock_rdlock(&_rwlock);
for (auto i : _file_list){
list.push_back(i.first);
}
pthread_rwlock_unlock(&_rwlock);
return true;
}
//通过文件名称获取文件对应的压缩包名称
bool GetFileGzip(std::string &file, std::string &gzip){
pthread_rwlock_rdlock(&_rwlock);
auto it = _file_list.find(file);
if (it == _file_list.end()){
pthread_rwlock_unlock(&_rwlock);
return false;
}
gzip = it->second;
pthread_rwlock_unlock(&_rwlock);
return true;
}
//向外提供获取文件数据功能
bool GetFileData(std::string &file, std::string &body)
{
if (bf::exists(file)){
//1. 非压缩文件数据获取
GetNormalFile(file, body);
}
else{
//2. 压缩文件数据获取
//获取压缩包名称 gzip
std::string gzip;
GetFileGzip(file, gzip);
UnCompressFile(gzip, file);
GetNormalFile(file, body);
}
}
bool AddFileRecord(const std::string file, const std::string &gzip){
pthread_rwlock_wrlock(&_rwlock);
_file_list[file] = gzip;
pthread_rwlock_unlock(&_rwlock);
}
bool SetFileData(const std::string &file, const std::string &body, const int64_t offset){
int fd = open(file.c_str(), O_CREAT | O_WRONLY, 0664);
if (fd < 0){
std::cerr << "open file " << file << " error\n";
return false;
}
flock(fd, LOCK_EX);
lseek(fd, offset, SEEK_SET);
int ret = write(fd, &body[0], body.size());
if (ret < 0){
std::cerr << "store file " << file << " data error\n";
flock(fd, LOCK_UN);
return false;
}
flock(fd, LOCK_UN);
close(fd);
AddFileRecord(file,"");
return true;
}
//因为这个是死循环,就该启动线程
//热度低的文件进行压缩存储
bool LowHeatFileStore(){
//1. 获取记录信息
GetListRecord();
while (1){
//2. 目录检测,文件压缩存储
//2.1. 获取list目录下文件名称
//2.2. 判断文件是否需要压缩
//2.3. 对文件进行压缩存储
//3. 存储记录信息
SetListRecord();
sleep(3);
}
}
};