Codes

server.cpp

#include "httplib.h"
#include <boost/filesystem.hpp>
#include <iostream>
#include <sstream>
#include <fstream>
#include "compress.hpp"

#define SERVER_BASE_DIR "www"
#define SERVER_ADDR "0.0.0.0"
#define SERVER_PORT 9000
#define SERVER_BACKUP_DIR SERVER_BASE_DIR"/list/"

using namespace httplib;
namespace bf = boost::filesystem;

CompressStore cstor;

class CloudServer{
private:
	SSLServer srv;
public:
	CloudServer(const char* cert, const char* key): srv(cert, key){
		bf::path base_path(SERVER_BASE_DIR);
		//如果文件不存在
		if (!bf::exists(base_path)){
			//创建目录
			bf::create_directory(base_path);
		}
		bf::path list_path(SERVER_BACKUP_DIR);
		if (!bf::exists(list_path)){
			bf::create_directory(list_path);
		}
		

	}
	bool Start(){
		srv.set_base_dir(SERVER_BASE_DIR);
		srv.Get("/(list(/){0,1}){0,1}", GetFileList);
		srv.Get("/list/(.*)", GetFileData);
		srv.Put("/list/(.*)", PutFileData);
		srv.listen("SERVER_ADDR", SERVER_PORT);
		return true;
	}
private:
	static void PutFileData(const Request &req, httplib::Response &rsp)
	{
		std::cout << "backup file " << req.path << "\n";
		if (!req.has_header("Range")){
			rsp.status = 400;
			return;
		}

			std::string range = req.get_header_value("Range");
			int64_t range_start;
			if (RangeParse(range, range_start) == false){
				rsp.status = 400;
				return;
			
		}
			
			std::string real = SERVER_BASE_DIR + req.path;
			cstor.SetFileData(real, req.body, range_start);
			std::ofstream file(real, std::ios::binary | std::ios::trunc);
			if (!file.is_open()){
				std::cerr << "open file " << real << "error\n";
				rsp.status = 500;
				return;
			}
			file.seekp(range_start, std::ios::beg);
			file.write(&req.body[0], req.body.size());
			if (!file.good()){
				std::cerr << "file write body error\n";
				rsp.status = 500;
				return;
			}
			file.close();
		return;
	}

	static bool RangeParse(std::string &range, int64_t &start){

		//Range: bytes=start-end
		size_t pos1 = range.find("=");
		size_t pos2 = range.find("=");
		if (pos1 == std::string::npos || pos2 == std::string::npos){
			std::cerr << "range:[" << range << "] format error\n";
			return false;
		}
		std::stringstream rs; 
		rs << range.substr(pos1 + 1, pos2 - pos1 - 1);
		rs >> start;
		return true;
	}
	//文件列表信息获取,定义成静态就是避免了this指针
	static void GetFileList(const Request &req, httplib::Response &rsp){
		
		std::vector<std::string> list;
		cstor.GetFileList(list);

		std::string body;
		body = "<html><body><ol><hr />";



		for (auto i:list){
			bf::path path(i);
			std::string file = path.filename().string();
			std::string uri = "/list/" + file;
			body += "<h4><li>";
			body += "<a href='";
			body += uri;
			body += "'>";
			body += file;
			body += "</a>";
			body += "</li></h4>";
			//std::string file = item_begin->path().string();
			//std::cerr << "file:" << file << std::endl;
		}
		body += "<hr /></ol></body></html>";
		rsp.set_content(&body[0], "text/html");
		return;
	}
	//获取文件数据(文件下载)
	static void GetFileData(const Request &req, httplib::Response &rsp){
		std::string real = SERVER_BASE_DIR + req.path;
		std::string  body;
		cstor.GetFileData(real, body);
		//正文只能给一次,文件不能太大
		rsp.set_content(body, "text/plain"); //plain文件下载
	}


};

void thr_start(){
	cstor.LowHeatFileStore();
}
int main()
{
	std::thread thr(thr_start);
	thr.detach();
	CloudServer srv("./cert.pem","./key.pem");
	srv.Start();
	return 0;
}

CloudClient.hpp

#define _D_SCL_SECURE_NO_WARNINGS 1
#ifndef __M_CLOUD_H__
#define __M_CLOUD_H__

#include <fstream>
#include <sstream>
#include <iostream>
#include <string>
#include <unordered_map>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string.hpp>
#include <thread>
#include "httplib.h"

#define SERVER_IP "192.168.122.132"
#define SERVER_PORT 9000
#define CLIENT_BACKUP_DIR ".\\backup"
#define CLIENT_BACKUP_INFO_FILE ".\\back.list"
#define RANGE_MAX_SIZE (10 << 20) //10MB
#define BACKUP_URI "/list/"

namespace bf = boost::filesystem;

class ThrBackUp
	//封装一个类,使用一个参数来传递
{
private:
	std::string _file;
	int64_t _range_start;
	int64_t _range_len;
public:
	bool _res;
public:
	ThrBackUp(const std::string &file, int64_t start, int64_t len) :_res(true),
	_file(file), _range_start(start), _range_len(len) {}
	void Start(){
		//获取文件的range分块数据
		std::stringstream ss;
		ss << "backup file" << _file << "range:[" << _range_start << "-" << _range_len << "\n";
		std::cout << ss.str();
		std::ifstream path(_file, std::ios::binary);
		if (!path.is_open()){
			std::cerr << "range backup file " << _file << "failed!\n";
			_res = false;
			return;
		}
		//跳转到range的起始位置
		path.seekg(_range_start, std::ios::beg);//跳转
		std::string body;
		
		body.resize(_range_len);
		//读取文件中range分块的文件数据
		path.read(&body[0], _range_len);
		if (!path.good()){
			std::cerr << "read file" << _file << " range data failed\n";
			_res = false;
			return;
		}
		path.close();
		//上传range数据
		bf::path name(_file);
		//组织上传的uri路径  method url version
		//PUT /list/filename HTTP/1.1
		std::string url = BACKUP_URI + name.filename().string();//native  不要双引号
		//实例化一个httplib的客户端对象
		httplib::Client cli(SERVER_IP, SERVER_PORT);
		//定义HTTP请求头信息
		httplib::Headers hdr;
		hdr.insert(std::make_pair("Content-Length", std::to_string(_range_len)));

		std::stringstream tmp;
		tmp << "bytes=" << _range_start << "-" << _range_start + _range_len - 1;
		hdr.insert(std::make_pair("Range", tmp.str().c_str()));
		//通过实例化的Client向服务端发送PUT请求
		auto rsp = cli.Put(url.c_str(), hdr, body, "text/plain");
		if (rsp &&rsp->status != 200){
			_res = false;
		}
		else {
			std::stringstream ss;
			ss << "backup file" << _file << "] range:[" << _range_start << "-" << _range_len << "] backup success\n";
			std::cout << ss.str();
			return;
		}
	}
};
class CloudClient
{
public:
	CloudClient(){
		bf::path file(CLIENT_BACKUP_DIR);
		if (!bf::exists(file)){
			bf::create_directory(file);
		}
	}
private:
	std::unordered_map<std::string, std::string> _backup_list;
	//CloudClient(){}
private:
	
	//获取备份信息
	bool GetBackupInfo()
	{
		//filename1 etag\n
		//filename2 etag\n
		//实例化path对象
		bf::path path(CLIENT_BACKUP_INFO_FILE);
		if (!bf::exists(path)){
			std::cerr << "list file" << path.string() << "is not exists\n";
			return false;
		}
		int64_t fsize = bf::file_size(path);
		if (fsize == 0){
			std::cerr << "have no backup info\n";
			return false;
		}
		std::string body;
		body.resize(fsize);
		std::ifstream file(CLIENT_BACKUP_INFO_FILE, std::ios::binary);
		if (!file.is_open()){
			std::cerr << "list file open error\n";
			return false;
		}
		file.read(&body[0], fsize);
		if (!file.good()){
			std::cerr << "read list file body error\n";
			return false;
		}
		file.close();
		std::vector<std::string> list;
		boost::split(list, body, boost::is_any_of("\n"));//切割后放到list中

		for (auto i : list){
			//filename2 etag
			size_t pos = i.find(" ");
			if (pos == std::string::npos)
				continue;
			std::string key = i.substr(0, pos);
			std::string val = i.substr(pos + 1);
			_backup_list[key] = val;
		}
		return true;
	}
	//将备份信息保存在指定文件中
	bool SetBackuoInfo()
	{
		std::string body;
		for (auto i : _backup_list){
			body += i.first + " " + i.second + "\n";
		}
		std::ofstream file(CLIENT_BACKUP_INFO_FILE, std::ios::binary);
		if (!file.is_open()){
			std::cerr << "open list file error\n";
			return false;
		}
		file.write(&body[0], body.size());
		if (!file.good()){
			std::cerr << "set backuo info error\n";
			return false;
		}
		file.close();
		return true;
	}
	//目录监控
	bool BackupDirListen(const std::string &path)
	{
		bf::directory_iterator item_begin(path);
		bf::directory_iterator item_end;
		for (; item_begin != item_end; ++item_begin){
			// /list/abc/abc.txt bc.txt
			if (bf::is_directory(item_begin->status())){
				BackupDirListen(item_begin->path().string());
				continue;
			}
			if (FileIsNeedBackup(item_begin->path().string()) == false){
				continue;
			}
			std::cerr << "file:[" << item_begin->path().string() << "need backup\n";
			if (PutFileData(item_begin->path().string()) == false){
				continue;
			}
			//上传成功后
			//记录文件Etag信息
			AddBackInfo(item_begin->path().string());

		}
		return true;
	}

	//添加一个信息
	bool AddBackInfo(const std::string &file)
	{
		//etag = "mtime-fsize"
		std::string etag;
		if (GetFileEtag(file, etag) == false)
			return false;
		_backup_list[file] = etag;
	}
	static void thr_start(ThrBackUp *backup_info)
	{
		backup_info->Start();
		std::cout << "into thr_start\n";
		return;
	}

	//备份文件
	bool PutFileData(const std::string &file)
	{
		//按大小对文件内容进行分块传输(10MB)
		//通过获取分块传输是否成功判断整个文件是否上传成功
		//选择多线程处理
		//1.获取文件大小
		int64_t fsize = bf::file_size(file);
		if (fsize <= 0){
			std::cerr << "file " << file << "unnecessary backup\n";
			return false;
		}
		//2.计算总共需要分多少块,得到每块大小以及起始位置
		//3.循环创建线程,在线程中上传文件数据
		int count = (int)fsize / RANGE_MAX_SIZE;
		std::vector<ThrBackUp>thr_res;
		std::vector<std::thread> thr_list;
		std::cerr << "file:[" << file << "]fsize:[" << fsize << "] count:[" << count + 1 << "]\n";
		// fsize = 500;
		for (int i = 0; i <= count; ++i){
			int64_t range_start = i * RANGE_MAX_SIZE; //起始位置
			int64_t range_end = (i + 1) * RANGE_MAX_SIZE - 1;
			if (i == count){
				range_end = fsize - 1;
			}
			int64_t range_len = range_end - range_start + 1;
			ThrBackUp backup_info(file, range_start, range_len);
			std::cerr << "file:[" << file << "] range:[" << range_start << "-" << range_end << "]-" << range_len << "\n";
			thr_res.push_back(backup_info);
			thr_list.push_back(std::thread(thr_start, &thr_res[i]));
			//std::thread thr(thr_start, file, range_start, range_len, &thr_res[i]);

		}
		/*for (int i = 0; i <= count; ++i)
		{
			thr_list.push_back(std::thread(thr_start, &thr_res[i]));
		}*/

		//4.等待所有线程退出,判断文件上传结果
		bool ret = true;
		for (int i = 0; i <= count; ++i){
			thr_list[i].join();
			if (thr_res[i]._res == true)
				continue;
		}	
		ret = false;
		//5.上传成功,则添加文件的备份信息记录。 native() //不要双引号
		if (ret == false){
			return false;
		}
			//AddBackInfo(file);
			std::cerr << "file:[" << file << "] backup success\n";
			return true;
	}
	
	bool FileIsNeedBackup(const std::string &file)
	{
		std::string etag;
		if (GetFileEtag(file, etag) == false)
			return false;
		auto it = _backup_list.find(file);
		if (it != _backup_list.end() && it->second == etag){
			//不需要备份
			return false;
		}
		
		return true;
	}
	bool GetFileEtag(const std::string &file, std::string &etag)
	{
		bf::path path(file);
		if (!bf::exists(path))
		{
			std::cerr << "get file" << file << "etag error\n";
			return false;
		}
		int64_t fsize = bf::file_size(path);
		int64_t mtime = bf::last_write_time(path);
		std::stringstream tmp;
		tmp << std::hex << fsize << "-" << std::hex << mtime;
		etag = tmp.str();
		return true;
	}
	
public:
	bool Start()
	{
		
		GetBackupInfo();//获取备份信息
		while (1)
		{
			//对文件目录进行监控
			BackupDirListen(CLIENT_BACKUP_DIR);
			SetBackuoInfo();
			Sleep(10);
		}
		return true;
	}
};
#endif

CloudClient.cpp

#include "CloudClient.hpp"

int main()
{
	CloudClient client;
	client.Start();
	return 0;
}

compress.hpp

#include <thread>
#include <boost/filesystem.hpp>
#include <fstream>
#include <boost/algorithm/string.hpp>
#include <sys/stat.h>
#include <zlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <pthread.h>
#include <sys/file.h>
#include <sstream>



#define UNGZIPFILE_PATH "www/list"
#define GZIPFILE_PATH "/www/zip"
#define RECORD_FILE "record.list"
#define HEAT_TIME    10

namespace bf = boost::filesystem;

class CompressStore{
private:
	std::string _file_dir;
	//用于保存文件列表
	std::unordered_map<std::string, std::string> _file_list; 
	pthread_rwlock_t_rwlock;
private:
	//1.每次压缩存储线程启动时候,从文件中读取列表信息
	bool GetListRecord()
	{
		//filename, gzipfilename
		bf::path name(RECORD_FILE);
		if (!bf::exists(name)){
			std::cerr << "record file is no exists\n";
			return false;
		}
		std::ifstream file(RECORD_FILE, std::ios::binary);
		if (!file.is_open()){
			std::cerr << "open record file read error\n";
			return false;
		}
		int64_t fsize = bf::file_size(name);
		std::string body;
		body.resize(fsize);
		file.read(&body[0], fsize);
		if (!file.good()){
			std::cerr << "record file body read error\n";
			return false;
		}
		file.close();

		std::vector<std::string> list;
		boost::split(list, body, boost::is_any_of("\n")); //切割
		for (auto i : list){
			//filename gzipname
			size_t pos = i.find(" ");
			if (pos == std::string::npos){
				continue;
			}
			std::string key = i.substr(0, pos);
			std::string val = i.substr(pos + 1);
			_file_list[key] = val;
		}
		return true;
	}
	//2.每次压缩存储完毕,都将列表信息存储到文件中
	bool SetListRecord(){
		std::stringstream tmp;
		for (auto i : _file_list){
			tmp << i.first << " " << i.second << "\n";
		}
		std::ofstream file(RECORD_FILE, std::ios::binary| std::ios::trunc);
		if (!file.is_open()){
			std::cerr << "record file open error\n";
			return false;
		}
		file.write(&tmp.str().c_str(), tmp.str().size());
		if (!file.good()){
			std::cerr << "record file write body error\n";
			return false;
		}
		file.close();
	}
	2.1. 对文件进行压缩存储
	//bool (){
	//	if (!bf::exists(UNGZIPFILE_PATH)){
	//		bf::create_directory(UNGZIPFILE_PATH);
	//	}
	//	bf::directory_iterator item_begin(UNGZIPFILE_PATH);
	//	bf::directory_iterator item_end;

	//	for (; item_begin != item_end; ++item_begin){
	//		if (bf::is_directory(item_begin->status)){
	//			continue;
	//		}
	//		std::string name = item_begin->path().string();
	//		if (!IsNeedCompress(name)){
	//			continue;
	//		}
	//		if (CompressFile(name)){
	//			std::cerr << "file:[" << name << "] store sucess\n";
	//		}
	//		else{
	//			std::cerr << "file:[" << name << "] store failed\n";
	//		}
	//	}
	//	return true;
	//}
	//2.2. 判断文件是否需要压缩
	bool IsNeedCompress(std::string &file){
		struct stat st;
		if (stat(file.c_str(), &st) < 0){
			std::cerr << "get file:[" << file << "] stat error\n";
			return false;
		}
		time_t cur_time = time(NULL);
		time_t acc_time = st.st_atime; //最后一次访问时间
		if ((cur_time - acc_time) < HEAT_TIME){
			return false;
		}
		return true;
	}
	//2.3. 对文件进行压缩存储
	bool CompressFile(std::string &file,std::string &gzip{
		int fd = open(file.c_str(), O_RDONLY);
		if (fd < 0){
			std::cerr << "com open file:[" << file << "] error\n";
			return false;
		}
		//std::string gzfile = file + ".gz";
		gzFile gf = gzopen(gzip.c_str(), "wb");
		if (gf == NULL){
			std::cerr << "com open gzip:[" << gzip << "] error\n";
			return false;
		}
		int ret;
		char buf[1024];
		flock(fd, LOCK_SH);
		while ((ret = read(fd, buf, 1024)) > 0){
			gzWrite(gf, buf, ret);
		}
		flock(fd, LOCK_UN);
		close(fd);
		gzclose(gf);
		unlink(file.c_str());
		return true;
	}
	//3. 对文件进行解压缩
	bool UnCompressFile(std::string &gzip,std::string &file{
		int fd = open(file.c_str(), O_CREAT | O_WRONLY, 0664);
		if (fd < 0){
			std::cerr << "open file" << file << " failed\n";
			return false;
		}
		gzFile gf = gzopen(gzip.c_str(), "rb");
		if (gf == NULL){
			std::cerr << "open gzip " << gzip << " failed\n";
			close(fd);
			return false;
		}
		int ret;
		char buf[1024] = { 0 };
		flock(fd, LOCK_EX);
		while ((ret = gzread(gf, buf, 1024)) > 0){
			int len = write(fd, buf, ret);
			if (len < 0){
				std::cerr << "get gzip data failed\n";
				gzclose(gf);
				close(fd);
				flock(fd, LOCK_UN);
				return false;
			}
		}
		flock(fd, LOCK_UN);
		gzclose(gf);
		close(fd);
		unlink(gzip.c_str());
		return true;
	}
	bool GetNormalFile(std::string &name, std::string &body){
		int64_t fsize = bf::file_size(name);
		body.resize(fsize);
		int fd = open(name.c_str(), O_RDONLY);
		if (fd < 0){
			std::cerr << "open file " << name << "failed\n";
			return false;
		}
		flock(fd, LOCK_SH);
		int ret = read(fd, &body[0], fsize);
		flock(fd, LOCK_UN);
		if (ret != fsize){
			std::cerr << "get file " << name << "body error\n";
			close(fd);
			return false;
		}
		close(fd);
		return true;
	}
	//目录检测,获取目录中的文件名
	//1. 判断文件是否需要压缩存储
	//2. 文件压缩存储
	bool DirectoryCheck(){
		if (bf::exists(UNGZIPFILE_PATH)){
			bf::create_directory(UNGZIPFILE_PATH);
		}
		bf::directory_iterator item_begin(UNGZIPFILE_PATH);
		bf::directory_iterator item_end;
		for (; item_begin != item_end; ++item_begin){
			if (bf::is_directory(item_begin->status())){
				continue;
			}
			std::string name = item_begin->path().string();
			std::string gzip = GZIPFILE_PATH+item_begin->path().filename().string() + ".gz";
			if (IsNeedCompress(name)){
				CompressFile(name, gzip);
				AddFileRecord(name, gzip);
			}
			
		}
		return true;

	}
public:
	CompressStore(){
		pthread_rwlock_inti(&_rwlock,NULL);
		if (!bf::exists(GZIPFILE_PATH)){
			bf::create_directory(GZIPFILE_PATH);
		}
	}

	~CompressStore(){
		pthread_rwlock_destroy(&_rwlock);
	}
	//向外提供获取文件列表功能
	bool GetFileList(std::vector<std::string> &list)
	{
		pthread_rwlock_rdlock(&_rwlock);
		for (auto i : _file_list){
			list.push_back(i.first);
		}
		pthread_rwlock_unlock(&_rwlock);
		return true;
	}

	//通过文件名称获取文件对应的压缩包名称
	bool GetFileGzip(std::string &file, std::string &gzip){
		pthread_rwlock_rdlock(&_rwlock);
		auto it = _file_list.find(file);
		if (it == _file_list.end()){
			pthread_rwlock_unlock(&_rwlock);
			return false;
		}
	    gzip = it->second;
		pthread_rwlock_unlock(&_rwlock);
		return true;
	}
	//向外提供获取文件数据功能
	bool GetFileData(std::string &file, std::string &body)
	{
		if (bf::exists(file)){
			//1. 非压缩文件数据获取
			GetNormalFile(file, body);
		}
		else{
			//2. 压缩文件数据获取
			//获取压缩包名称 gzip
			std::string gzip;
			GetFileGzip(file, gzip);
			UnCompressFile(gzip, file);
			GetNormalFile(file, body);
		}	
	}

	bool AddFileRecord(const std::string file, const std::string &gzip){
		pthread_rwlock_wrlock(&_rwlock);
		_file_list[file] = gzip;
		pthread_rwlock_unlock(&_rwlock);
	}
	bool SetFileData(const std::string &file, const std::string &body, const int64_t offset){
		int fd = open(file.c_str(), O_CREAT | O_WRONLY, 0664);
		if (fd < 0){
			std::cerr << "open file " << file << " error\n";
			return false;
		}
		flock(fd, LOCK_EX);
		lseek(fd, offset, SEEK_SET);
		int ret = write(fd, &body[0], body.size());
		if (ret < 0){
			std::cerr << "store file " << file << " data error\n";
			flock(fd, LOCK_UN);
			return false;
		}
		flock(fd, LOCK_UN);
		close(fd);
		AddFileRecord(file,"");
		return true;
	}

	//因为这个是死循环,就该启动线程
	//热度低的文件进行压缩存储
	bool LowHeatFileStore(){ 
		//1. 获取记录信息
		GetListRecord();
		while (1){
			//2. 目录检测,文件压缩存储
				//2.1. 获取list目录下文件名称
				//2.2. 判断文件是否需要压缩
				//2.3. 对文件进行压缩存储
			//3. 存储记录信息
			SetListRecord();
			sleep(3);

		}
		
	}
};
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值