云备份

最新推荐文章于 2021-08-30 00:04:02 发布

yl_sakura

最新推荐文章于 2021-08-30 00:04:02 发布

阅读量315

点赞数

本文链接：https://blog.csdn.net/yl_sakura/article/details/100184225

版权

项目介绍

该项目完成一个自动对指定目录下的文件进行备份的功能；

客户端对指定目录进行监控，对每个文件生成etag信息，鉴别是否需要备份
客户端将需要备份的文件基于http协议的PUT请求上传到服务器端
服务端对于PUT上传的文件进行备份到指定目录下
整个通信的过程使用SSL/TLS加密传输

服务端设计

实现功能：

提供解析基于https协议的put请求，将文件数据进行备份
提供浏览器能够查看服务器上文件信息功能
提供浏览器能够下载服务器上文件功能
提供对后台长时间无访问文件的压缩存储功

服务端代码：

运行环境：Linux平台

  1 #include "httplib.h"
  2 #include <fstream>
  3 #include <unistd.h>
  4 #include <fcntl.h>
  5 #include <boost/filesystem.hpp>
  6 
  7 #define SERVER_BASE_DIR "www"
  8 #define SERVER_BACKUP_DIR SERVER_BASE_DIR"/list/"
  9 #define SERVER_ADDR "0.0.0.0"
 10 #define SERVER_PORT 9527
 11 
 12 namespace bf = boost::filesystem;
 13 
 14 class CloudServer
 15 {
 16     private:
 17         httplib::Server srv;
 18     public:
 19         CloudServer()
 20         {
 21             bf::path base_path(SERVER_BASE_DIR);
 22             if(!bf::exists(base_path)){
 23                 bf::create_directory(base_path);
 24             }
 25             bf::path list_path(SERVER_BACKUP_DIR);
 26             if(!bf::exists(list_path)){
 27                 bf::create_directory(list_path);
 28             }
 29         }
 30         bool Start(){
 31             srv.set_base_dir(SERVER_BASE_DIR);
 32             srv.Get("/(list(/){0,1}){0,1}", GetFileList);
 33             srv.Get("/list/(.*)", GetFileData);
 34             srv.Put("/list/(.*)", PutFileData);
 35             srv.listen(SERVER_ADDR, SERVER_PORT);
 36             return true;
 37         }
 38     private:
 39         static void GetFileList(const httplib::Request &req, httplib::Response &rsp)
 40         {
 41             bf::path list(SERVER_BASE_DIR);
 42             bf::directory_iterator item_begin(list);
 43             bf::directory_iterator item_end;
 44 
 45             std::string body;
 46             body = "<html><body><ol><hr />";
 47             for(; item_begin != item_end; ++item_begin){
 48                 if(bf::is_directory(item_begin->status())){
 49                     continue;
 50                 }
 51                 std::string file = item_begin->path().filename().string();
 52                 std::string uri = "/list/" + file;
 53                 body += "<h4><li>";
 54                 body += "<a href='";
 55                 body += uri;
 56                 body += "'>";
 57                 body += file;
 58                 body += "</a>";
 59                 body += "</li></h4>";
 60             }
 61             body += "<hr /></ol></body></html>";
 62             rsp.set_content(&body[0], "text/html");
 63             return;
 64         }
 65         static void GetFileData(const httplib::Request &req, httplib::Response &rsp)
 66         {
 67             std::string file = SERVER_BASE_DIR + req.path;
 68             if(!bf::exists(file)){
 69                 std::cerr << "file" << file << "is not exists";
 70                 rsp.status = 404;
 71                 return;
 72          }
 73          std::ifstream ifile(file, std::ios::binary);
74             if(!ifile.is_open()){
 75                 std::cerr << "open file " << file << "error\n";
 76                 rsp.status = 500;
 77                 return ;
 78             }
 79             std::string body;
 80             int64_t fsize = bf::file_size(file);
 81             body.resize(fsize);
 82             ifile.read(&body[0], fsize);
 83             if(!ifile.good()){
 84                 std::cerr << "read file" << file << " body error\n";
 85                 rsp.status = 500;
 86                 return ;
 87             }
 88             rsp.set_content(body, "text/html");
 89         }
 90         static void PutFileData(const httplib:: Request &req, httplib::Response &rsp)
 91         {
 92             if(!req.has_header("Range")){
 93                 rsp.status = 400;
 94                 return ;
 95             }
 96             std::string range = req.get_header_value("Range");
 97             int64_t range_start;
 98             if(RangeParse(range, range_start) == false){
 99                 rsp.status = 400;
100                 return ;
101             }
102             std::cout << "backup file:[" << req.path << "]range:[" << range << "]\n";
103             std::string real = SERVER_BASE_DIR + req.path;
104             int fd = open(real.c_str(),O_CREAT|O_WRONLY, 0664);
105             if(fd < 0){
106                 std::cerr << "open file" << real << " error\n";
107                 rsp.status = 500;
108                 return ;
109          	}
110             lseek(fd, range_start, SEEK_SET);
111             int ret = write(fd, &req.body[0], req.body.size());
112             if(ret != req.body.size()){
113                 std::cerr << "file write body error\n";
114                 rsp.status = 500;
115                 return ;
116             }
117             close(fd);
118 
119             return ;
120         }
121         static bool RangeParse(std::string &range, int64_t &start){
122             size_t pos1 = range.find("=");
123             size_t pos2 = range.find("-");
124             if(pos1 == std::string::npos || pos2 == std::string ::npos){
125                 std::cerr << "Range:[" << range << "] format error\n";
126                 return false;
127             }
128             std::stringstream rs;
129             rs << range.substr(pos1+1, pos2 - pos1 - 1);
130             rs >> start;
131             return true;
132         }
138 };
140 int main()
141 {
142     CloudServer srv;
143     srv.Start();
144     return 0;
145 }

客户端设计

实现功能：

提供监控目录的功能，能够获取目录下文件信息，鉴别文件是否需要备份
备份文件，基于HTTPS协议PUT请求，实现文件多线程分块上传功能
文件的信息记录，便于文件是否需要备份的鉴别

客户端代码：

运行环境：vs2015

#ifndef __M_CLOUD_H__
#define __M_CLOUD_H__
#include <iostream>
#include <fstream>
#include <sstream>
#include <string>
#include <vector>
#include <thread>
#include <unordered_map>
#include <boost\filesystem.hpp>
#include <boost\algorithm\string.hpp>
#include "httplib.h"

#define CLIENT_BACKUP_DIR "backup"
#define CLIENT_BACKUP_INFO_FILE "back.list"
#define RANGE_MAX_SIZE (10 << 20)
#define SERVER_IP "192.168.194.129"
#define SERVER_PORT 9527
#define BACKUP_URI "/list/"

namespace bf = boost::filesystem;

class ThrBackup
{
private:
	std::string _file;
	int64_t _range_start;
	int64_t _range_len;
public:
	bool _res;
public:
	ThrBackup(const std::string &file, int64_t start, int64_t len):_res(true),
		 _file(file), _range_start(start), _range_len(len){}
	void Start() {
		//获取文件的range分块数据
		std::ifstream path(_file, std::ios::binary);
		if (!path.is_open()) {
			std::cerr << "range backup file" << _file << "failed\n";
			_res = false;
			return;
		}
		//跳转到range的起始位置
		path.seekg(_range_start, std::ios::beg);
		std::string body;
		body.resize(_range_len);
		//读取文件中range分块的文件数据
		path.read(&body[0], _range_len);
		if (!path.good()) {
			std::cerr << "read file " << _file << "range data failed\n";
			_res = false;
			return;
		}
		path.close();

		//上传range数据
		bf::path name(_file);
		//组织上传的url路径
		std::string uri = BACKUP_URI + name.filename().string();
		//实例化一个httplib的客户端对象
		httplib::Client cli(SERVER_IP, SERVER_PORT);
		httplib::Headers hdr;
		hdr.insert(std::make_pair("Content-Length", std::to_string(_range_len)));
		std::stringstream tmp;
		tmp << "bytes=" << _range_start << "-" << (_range_start + _range_len - 1);
		hdr.insert(std::make_pair("Range", tmp.str().c_str()));
		//通过实例化的Client想服务端
		auto rsp = cli.Put(uri.c_str(), hdr, body, "text/plain");
		if (rsp && rsp->status != 200) {
			_res = false;
		}
		std::stringstream ss;
		ss << "backup file" << _file << "range:[" << _range_start << "-" << _range_len << "] backup success\n";
		std::cout << ss.str();
		return ;
	}
};

class CloudClient
{
private:
	std::unordered_map<std::string, std::string> _backup_list;
private:
	bool GetBackupInfo()
	{
		bf::path path(CLIENT_BACKUP_INFO_FILE);
		if (!bf::exists(path) ){
			std::cerr << "list file" << path.string() << "is not exist" << std::endl;
			return false;
		}
		int64_t fsize = bf::file_size(path);
		if (fsize == 0) {
			std::cerr << "have no backup info\n";
			return false;
		}
		std::string body;
		body.resize(fsize);
		std::ifstream file(CLIENT_BACKUP_INFO_FILE, std::ios::binary);
		if (!file.is_open()) {
			std::cerr << "list file open error\n";
			return false;
		}
		file.read(&body[0], fsize);
		if (!file.good()) {
			std::cerr << "read list file body error\n";
			return false;
		}
		file.close();
		std::vector<std::string> list;
		boost::split(list, body,boost::is_any_of ("\n"));
		for (auto i : list) {
			size_t pos = i.find(" ");
			if (pos == std::string::npos) {
				continue;
			}
			std::string key = i.substr(0, pos);
			std::string val = i.substr(pos + 1);
			_backup_list[key] = val;
		}
		return true;
	}
	bool SetBackupInfo()
	{
		std::string body;
		for (auto i : _backup_list) {
			body += i.first + " " + i.second + "\n";
		}
		std::ofstream file(CLIENT_BACKUP_INFO_FILE, std::ios::binary);
		if (!file.is_open()) {
			std::cerr << "open list file error\n";
			return false;
		}
		file.write(&body[0], body.size());
		if (!file.good()) {
			std::cerr << "set backup info error\n";
			return false;
		}
		file.close();
		return true;
	}
	bool BackupDirListen(const std::string &path)
	{
		bf::directory_iterator item_begin(path);
		bf::directory_iterator item_end;
		for (; item_begin != item_end; ++item_begin) {
			if (bf::is_directory(item_begin->status())) {
				BackupDirListen(item_begin->path().string());
				continue;
			}
			if (FileIsNeedBackup(item_begin->path().string()) == false) {
				continue;
			}
			if (PutFileData(item_begin->path().string()) == false) {
				continue;
			}
			AddBackupInfo(item_begin->path().string());
		}
		return true;
	}
	bool AddBackupInfo(const std::string &file)
	{
		std::string etag;
		if (GetFileEtag(file, etag) == false) {
			return false;
		}
		_backup_list[file] = etag;
	}
	bool GetFileEtag(const std::string &file, std::string &etag)
	{
		bf::path path(file);
		if (!bf::exists(path)) {
			std::cerr << "get file" << file << "etag error\n";
			return false;
		}
		int64_t fsize = bf::file_size(path);
		int64_t mtime = bf::last_write_time(path);
		std::stringstream tmp;
		tmp << std::hex << fsize << "-" << std::hex << mtime;
		etag = tmp.str();
		return 0;
	}
	bool PutFileData(const std::string &file)
	{
		//1.获取文件大小
		int64_t fsize = bf::file_size(file);
		if (fsize <= 0) {
			std::cerr << "file" << file << "unnecessary backup\n";
			return false;
		}
		//2.计算总共需要分多少块，得到每块大小以及起始位置
		//3.循环创建线程，在线程中上传文件数据
		int count = fsize / RANGE_MAX_SIZE;
		std::vector<ThrBackup> thr_res;
		std::vector<std::thread> thr_list;
		std::cerr << "file:[" << file << "] fsize:[" << fsize << "] count:[" << count + 1 << "]\n";
		for (int i = 0; i <= count; i++) {
			int64_t range_start = i * RANGE_MAX_SIZE;
			int64_t range_end = (i + 1) * RANGE_MAX_SIZE - 1;
			if (i == count) {
				range_end = fsize - 1;
			}
			int64_t range_len = range_end - range_start + 1;
			ThrBackup backup_info(file, range_start, range_len);
			std::cerr << "file:[" << file << "]range:[" << range_start << "-" << range_end << "]-" << range_len << "\n";
			thr_res.push_back(backup_info);
		}
		for (int i = 0; i <= count; i++) {
			thr_list.push_back(std::thread(thr_start, &thr_res[i]));
		}
		//4.等待线程退出，判断文件上传结果
		bool ret = true;
		for (int i = 0; i <= count; i++) {
			thr_list[i].join();
			if (thr_res[i]._res == true) {
				continue;
			}
			ret = false;
		}
		//5.上传成功，则添加文件的备份信息记录
		if (ret == false) {
			return false;
		}
		
		std::cerr << "file:[" << file << "] backup success\n";
		return true;
	}
	bool FileIsNeedBackup(const std::string &file)
	{
		std::string etag;
		if (GetFileEtag(file, etag) == false) {
			return false;
		}
		auto it = _backup_list.find(file);
		if (it != _backup_list.end() && it->second == etag) {
			return false;
		}
		return true;
	}
	static void thr_start(ThrBackup *backup_info) {
		backup_info->Start();
		return;
	}
public:
	bool Start() {
		GetBackupInfo();
		while (1) {
			BackupDirListen(CLIENT_BACKUP_DIR);
			SetBackupInfo();
			Sleep(10);
		}
		return true;
	}
};

#endif

httplib基本使用

/*test.cpp*/ #include "httplib.h"
static void HelloWorld(const httplib::Request &req, httplib::Response &rsp)
 {    
 	rsp.set_content("<html>hello world</html>", "text/html");    return;
 }
int main()
{    
	httplib::Server srv;  
	srv.set_base_dir("./www");    
	srv.Get("/", HelloWorld);    
	srv.listen("0.0.0.0", 9000); 
} 
	/*g++ -std=c++0x test.cpp -o test -lpthread -lboost_filesystem -lboost_system*/

boost库下载路径：boost_1_70_0-msvc-14.1-64.exe

yl_sakura

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
云备份

项目介绍该项目完成一个自动对指定目录下的文件进行备份的功能；客户端对指定目录进行监控，对每个文件生成etag信息，鉴别是否需要备份客户端将需要备份的文件基于http协议的PUT请求上传到服务器端服务端对于PUT上传的文件进行备份到指定目录下整个通信的过程使用SSL/TLS加密传输服务端设计实现功能：提供解析基于https协议的put请求，将文件数据进行备份提供浏览器能够查看...
复制链接

扫一扫