使用libCurl从Web服务器下载文件

XlibCurl有两种下载文件方式，同步方式使用easy接口的curl_easy_perform，异步方式可以同时下载多个文件，使用multi接口的curl_multi_perform，通过 curl_multi_wait 等待传输事件的发生，curl_multi_info_read读取异步传输中的状态。

本例使用http下载单个文件和多个文件，ftp方式上传和下载文件，ftp支持断点续传。

1、头文件和数据结构定义：

#include <iostream>
#include <vector>
#include <string_view>
#include <fstream>
#include <iostream>
#include <filesystem>

#include "curl/curl.h"

#pragma comment(lib,"ws2_32.lib")
#pragma comment(lib,"libcurl.lib")

#define MAXCONNECTS 3 //同时下载的文件数据

using namespace std;

typedef struct curl_obj
{
	CURL* curl_handle = NULL;
	FILE* save_file = NULL;
	std::string fetch_url;
	size_t transfer_size = 0; //上传、下载字节数
	long file_size = 0;

	curl_off_t lastruntime;//下载进度

	size_t(*recive_data_fun)(void* ptr, size_t size, size_t nmemb, void* stream);
	size_t(*read_head_fun)(void* ptr, size_t size, size_t nmemb, void* stream);
}curl_obj;

2、libcurl的回调函数

//将下载内容写入文件
size_t recive_data_func(void* ptr, size_t size, size_t nmemb, void* obj)
{
	curl_obj* buf = (curl_obj*)obj;
	size_t writesize = fwrite(ptr, size, nmemb, buf->save_file);
	buf->transfer_size += writesize;
	fflush(buf->save_file);
	return writesize;
}

//将下载内容写入缓存
static size_t write_data(void* contents, size_t size, size_t nmemb, void* stream)
{
	size_t written = ((size_t*)stream)[0];//上次已经下载的内容
	size_t realsize = size * nmemb;
	((size_t*)stream)[0] = written + realsize;
	char* mem = ((char*)stream) + sizeof(size_t) + written;
	memcpy(mem, contents, realsize);
	return realsize;
}

//读取文件用于上传
static size_t read_data_func(void* ptr, size_t size, size_t nmemb, void* obj)
{
	curl_obj* buf = (curl_obj*)obj;
	if (ferror(buf->save_file))
	{
		return CURL_READFUNC_ABORT;
	}
	size_t n = fread(ptr, size, nmemb, buf->save_file) * size;
	buf->transfer_size += n;
	return n;
}

size_t read_head_fun(void* ptr, size_t size, size_t nmemb, void* stream)
{
	char head[2048] = { 0 };
	memcpy(head, ptr, size * nmemb + 1);
	printf(" %s \n", head);
	return size * nmemb;
}

//传输进度
static int xfer_info_func(void* p,
	curl_off_t dltotal, curl_off_t dlnow,
	curl_off_t ultotal, curl_off_t ulnow)
{
	curl_obj* myp = (curl_obj*)p;
	CURL* curl = myp->curl_handle;
	curl_off_t curtime = 0;

	curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME_T, &curtime);

	/* under certain circumstances it may be desirable for certain functionality
	   to only run every N seconds, in order to do this the transaction time can
	   be used */
	if ((curtime - myp->lastruntime) >= 3000000)
	{
		myp->lastruntime = curtime;
		fprintf(stderr, "TOTAL TIME: %" CURL_FORMAT_CURL_OFF_T ".%06ld\r\n",
			(curtime / 1000000), (long)(curtime % 1000000));
	}

	if (ultotal)
	{		
		printf("UP progress: %0.2f%%\n", (float)ulnow*100 / ultotal);
	}
	if (dltotal)
	{		
		printf("DOWN progress: %0.2f%%\n", (float)dlnow*100 / dltotal);
	}
	return 0;
}

3、分割字符串的函数

//将string按分隔符，分成列表
std::vector<std::string_view> Split(std::string_view sv, char ch)
{
	std::vector<std::string_view> points;
	size_t point = 0;
	for (size_t i = 0; i < sv.length(); i++)
	{
		if (sv[i] == ch)
		{
			points.emplace_back(std::string_view(&sv[point], i - point));
			point = i + 1;
		}
	}
	if (point >= sv.size())
	{
		points.emplace_back(std::string_view(&sv[point - 1], sv.size() - point));
	}
	else {
		points.emplace_back(std::string_view(&sv[point], sv.size() - point));
	}
	return points;
}

4、获取远程文件大小，http和ftp方式不一样，目前还不能统一

//获得http文件大小
long GetHttpFileLength(std::string url)
{
	double downloadFileLenth = 0;

	CURL* handle = curl_easy_init();
	curl_easy_setopt(handle, CURLOPT_URL, url.c_str());
	curl_easy_setopt(handle, CURLOPT_HEADER, 0);    //只需要header头
	//curl_easy_setopt(handle, CURLOPT_VERBOSE, 1L);
	curl_easy_setopt(handle, CURLOPT_NOBODY, 1);    //不需要body

	if (curl_easy_perform(handle) == CURLE_OK)
	{
		curl_easy_getinfo(handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &downloadFileLenth);
	}
	else
	{
		cout << "get file size error " << url << endl;
		downloadFileLenth = -1;
	}
	curl_easy_cleanup(handle);
	return downloadFileLenth;
}

//获得ftp文件大小
long GetFtpFileLength(std::string url)
{
	long downloadFileLenth = -1;
	string baseurl = url.substr(0, url.find_last_of("/")+1);
	string filename = url.substr(url.find_last_of("/") + 1);

	CURL* handle = curl_easy_init();
	curl_easy_setopt(handle, CURLOPT_URL, baseurl.c_str());
	

	curl_easy_setopt(handle, CURLOPT_USERPWD, "ftp:123456");        //ftp用户名和密码
	curl_easy_setopt(handle, CURLOPT_CUSTOMREQUEST, ("LIST " + filename).c_str());
	//MLSD 只能列目录，不能获取单个文件大小
	curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, write_data);
	char buff[1024];
	memset(buff, 0, 1024);
	curl_easy_setopt(handle, CURLOPT_WRITEDATA, (void*)buff);
	if (curl_easy_perform(handle) == CURLE_OK)
	{
		size_t readsize = ((size_t*)buff)[0];
		char* p = buff + sizeof(size_t);
		string content = p;
		vector<string> list;
		
		for (auto& node : Split(content, ' '))
		{
			if (node.size() > 0)
				list.push_back(string(node));
		}
		if (list.size() >= 4)
			downloadFileLenth = std::atol(list[4].c_str());
	}
	else
	{
		cout << "get file size error " << url << endl;
		downloadFileLenth = -1;
	}
	curl_easy_cleanup(handle);
	return downloadFileLenth;
}

5、通过http下载文件

//下载一个HTTP文件
bool http_download(std::string pageuri, std::string filename)
{
	long orisize = GetHttpFileLength(pageuri);//获得文件大小。
	if (orisize != -1)
	{
		curl_obj obj;
		obj.curl_handle = curl_easy_init();
		if (!obj.curl_handle)
			return false;
		obj.fetch_url = pageuri;
		obj.file_size = orisize;//原始文件大小
		obj.save_file = fopen(filename.c_str(), "wb");
		if (!obj.save_file) {
			std::cout << "ERROR----fail!!!" << std::endl;
			curl_easy_cleanup(obj.curl_handle);
			return false;
		}
		else
		{		
			curl_easy_setopt(obj.curl_handle, CURLOPT_VERBOSE, 0L);
			curl_easy_setopt(obj.curl_handle, CURLOPT_HEADER, 0);
			curl_easy_setopt(obj.curl_handle, CURLOPT_NOPROGRESS, 1L);
			curl_easy_setopt(obj.curl_handle, CURLOPT_NOSIGNAL, 1L);
			curl_easy_setopt(obj.curl_handle, CURLOPT_URL, obj.fetch_url.c_str());
			curl_easy_setopt(obj.curl_handle, CURLOPT_WRITEDATA, (void*)&obj);
			curl_easy_setopt(obj.curl_handle, CURLOPT_WRITEFUNCTION,recive_data_func);             
#ifdef CURL_DEBUG 
			//curl_easy_setopt(obj[i].curl_handle, CURLOPT_VERBOSE, 1);
//#else 
			curl_easy_setopt(obj.curl_handle, CURLOPT_HEADERFUNCTION, /*obj[i].*/read_head_fun);
#endif  
			//下载
			if (curl_easy_perform(obj.curl_handle) != CURLE_OK)
			{
				curl_easy_cleanup(obj.curl_handle);
				fclose(obj.save_file);
				return false;
			}
			else
			{
				curl_easy_cleanup(obj.curl_handle);
				fclose(obj.save_file);
				return obj.transfer_size == obj.file_size;
			}
		}
	}
	else
		return false;
}

//初始化下载任务
bool init_multi_curl(curl_obj* obj, unsigned int urls_index, const std::vector<std::string>& pageuris,
	const std::vector <std::string>& localfiles)
{
	obj[urls_index].curl_handle = curl_easy_init();
	if (!obj[urls_index].curl_handle)
	{
		printf("fetch [%s]----ERROR!!!\n", pageuris[urls_index].c_str());
		return false;
	}
	obj[urls_index].fetch_url = pageuris[urls_index];
	obj[urls_index].file_size = GetHttpFileLength(obj[urls_index].fetch_url);//原始文件大小

	std::string out_filename = localfiles[urls_index];
	printf("----save_file[%d] [%s]\n", urls_index, out_filename.c_str());
	obj[urls_index].save_file = fopen(out_filename.c_str(), "wb");
	if (!obj[urls_index].save_file) 
	{
		curl_easy_cleanup(obj[urls_index].curl_handle);
		printf("Open file ERROR-- [%s] --fail!!!\n", out_filename.c_str());
		return false;
	}
	else
	{
		curl_easy_setopt(obj[urls_index].curl_handle, CURLOPT_VERBOSE, 0L);
		curl_easy_setopt(obj[urls_index].curl_handle, CURLOPT_HEADER, 0);
		curl_easy_setopt(obj[urls_index].curl_handle, CURLOPT_NOPROGRESS, 1L);
		curl_easy_setopt(obj[urls_index].curl_handle, CURLOPT_NOSIGNAL, 1L);
		curl_easy_setopt(obj[urls_index].curl_handle, CURLOPT_URL, obj[urls_index].fetch_url.c_str());//set down load url 
		curl_easy_setopt(obj[urls_index].curl_handle, CURLOPT_WRITEDATA, (void*) & obj[urls_index]);//set download file 
		curl_easy_setopt(obj[urls_index].curl_handle, CURLOPT_WRITEFUNCTION,/* obj[i].*/recive_data_func);//set call back fun             
#ifdef CURL_DEBUG 
		//curl_easy_setopt(obj[i].curl_handle, CURLOPT_VERBOSE, 1);
//#else 
		curl_easy_setopt(obj[urls_index].curl_handle, CURLOPT_HEADERFUNCTION, /*obj[i].*/read_head_fun);//set call back fun 
#endif  
		return true;
	}
}

//下载多个HTTP文件
bool http_multi_download(const std::vector<std::string>& pageuris, const std::vector <std::string>& localfiles)
{
	bool download_status = true;
	curl_obj* obj = new curl_obj[pageuris.size()];
	memset(obj,0, sizeof(curl_obj) * pageuris.size());
	//size_t mulit_h_num = pageuris.size();//总数量

	CURLM* multi_handle = curl_multi_init();
	if (!multi_handle)
	{
		delete[] obj;
		return false;
	}
	unsigned int urls_index = 0; //下载列表的序号
	for (urls_index = 0; urls_index < MAXCONNECTS; ++urls_index)
	{
		if (init_multi_curl(obj, urls_index, pageuris, localfiles))
			curl_multi_add_handle(multi_handle, obj[urls_index].curl_handle);//add task 
		else
		{
			cout << "init error " << pageuris[urls_index] << endl;
			download_status = false;
		}
	}

	int still_running, repeats = 0;
	CURLMcode mc;
	int numfds;
	CURLMsg* msg = NULL;
	do {	
		mc = curl_multi_perform(multi_handle, &still_running);
		if (mc == CURLM_OK) {
			/* wait for activity, timeout or "nothing" */
			mc = curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
		}
		if (mc != CURLM_OK) {
			fprintf(stderr, "curl_multi failed, code %d.\n", mc);
			download_status = false;
			break;
		}
		/* 'numfds' being zero means either a timeout or no file descriptors to
		   wait for. Try timeout on first occurrence, then assume no file
		   descriptors and no file descriptors to wait for means wait for 100
		   milliseconds. */
		if (!numfds)
		{
			repeats++; /* count number of repeated zero numfds */
			if (repeats > 1)
			{
				Sleep(1000);
			}
		}
		else
			repeats = 0;
		//检查文件传输结束
		int msgs_left = 0;		
		do {
			msg = curl_multi_info_read(multi_handle, &msgs_left);
			if (msg && (msg->msg == CURLMSG_DONE))
			{
				CURL* e = msg->easy_handle;
				long http_response_code = -1;
				char* http_url = NULL;
				curl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &http_response_code);
				curl_easy_getinfo(msg->easy_handle, CURLINFO_EFFECTIVE_URL, &http_url);

				//一个文件下载结束
				for (int j = 0; j < pageuris.size(); j++)
				{
					if (obj[j].curl_handle == e)
					{
						fflush(obj[j].save_file);
						fclose(obj[j].save_file);
						printf("[%d][%s] fetch done, response[%d] ,file size [%ld] ,download size [%zd]\n", j, http_url, http_response_code, obj[j].file_size, obj[j].transfer_size);

						curl_multi_remove_handle(multi_handle, obj[j].curl_handle);
						curl_easy_cleanup(obj[j].curl_handle);
						obj[j].curl_handle = NULL;

						if (obj[j].transfer_size != obj[j].file_size)
						{
							std::cout << obj[j].fetch_url << " 下载故障： " << obj[j].transfer_size << " / " << obj[j].file_size << endl;;
							//下载有问题,可能需要重新下载？
							download_status = false;
						}
						break;
					}
				}
				//添加新的下载文件
				if (urls_index < pageuris.size())
				{
					if (init_multi_curl(obj, urls_index, pageuris, localfiles))
						curl_multi_add_handle(multi_handle, obj[urls_index].curl_handle);//add task 
					else
					{
						cout << "init error " << pageuris[urls_index] << endl;
						download_status = false;
					}
					urls_index++;
					still_running++;				
				}
			}
		} while (msg);
		//继续下载
	} while (still_running);

	curl_multi_cleanup(multi_handle);

	delete[] obj;
	obj = NULL;
	return download_status;
}

6、通过ftp上传和下载文件

//上传一个FTP文件
int ftp_upload(std::string pageuri, std::string localpath_file, long timeout)
{
	long uploaded_len = GetFtpFileLength(pageuri); //已经上传文件的大小
	if (uploaded_len == -1)//文件不存在
		uploaded_len = 0;

	curl_obj obj;
	obj.curl_handle = curl_easy_init();
	if (!obj.curl_handle)
		return false;
		
	std::filesystem::path localpath(localpath_file);
	obj.file_size = std::filesystem::file_size(localpath);

	obj.save_file = fopen(localpath_file.c_str(), "rb");
	if (!obj.save_file)
	{
		std::cout << "ERROR----fail!!!" << std::endl;
		curl_easy_cleanup(obj.curl_handle);
		return false;
	}

	curl_easy_setopt(obj.curl_handle, CURLOPT_UPLOAD, 1L);               //使能上传功能
	curl_easy_setopt(obj.curl_handle, CURLOPT_URL, pageuri.c_str());        //上传目录
	curl_easy_setopt(obj.curl_handle, CURLOPT_USERPWD, "ftp:123456");        //ftp用户名和密码
	curl_easy_setopt(obj.curl_handle, CURLOPT_READFUNCTION, read_data_func);   
	curl_easy_setopt(obj.curl_handle, CURLOPT_READDATA,(void*) & obj);
	curl_easy_setopt(obj.curl_handle, CURLOPT_NOPROGRESS, 0L);
	curl_easy_setopt(obj.curl_handle, CURLOPT_VERBOSE, 0L);
	curl_easy_setopt(obj.curl_handle, CURLOPT_FTP_CREATE_MISSING_DIRS, 1);//自动创建文件夹
	curl_easy_setopt(obj.curl_handle, CURLOPT_INFILESIZE_LARGE, obj.file_size - uploaded_len);//上传文件大小
	curl_easy_setopt(obj.curl_handle, CURLOPT_XFERINFOFUNCTION, xfer_info_func);
	curl_easy_setopt(obj.curl_handle, CURLOPT_XFERINFODATA, (void*)&obj);
	if (timeout)
	{
		curl_easy_setopt(obj.curl_handle, CURLOPT_FTP_RESPONSE_TIMEOUT, timeout);
	}	
	//curl_easy_setopt(obj.curl_handle,  CURLOPT_HEADERFUNCTION, getContentLengthFunc);
	//curl_easy_setopt(obj.curl_handle, CURLOPT_HEADERDATA, &uploaded_len);	

	CURLcode ret = CURLE_GOT_NOTHING;
	if (uploaded_len)
	{
		fseek(obj.save_file, uploaded_len, SEEK_SET);
		curl_easy_setopt(obj.curl_handle, CURLOPT_APPEND, 1L);
	}	
	ret = curl_easy_perform(obj.curl_handle);

	//上传
	if (ret != CURLE_OK)
	{
		curl_easy_cleanup(obj.curl_handle);
		fclose(obj.save_file);
		return false;
	}
	else
	{
		curl_easy_cleanup(obj.curl_handle);
		fclose(obj.save_file);
		return (obj.transfer_size + uploaded_len) == obj.file_size;
	}
	return true;
}

bool ftp_download(std::string pageuri, std::string localpath_file, long timeout)
{
	curl_obj obj;
	obj.file_size = GetFtpFileLength(pageuri);//需要下载文件的大小。可能小于文件实际大小（续传）
	if (obj.file_size == -1)
	{
		cout << pageuri << "  file not exist!" << endl;
		return false;
	}

	obj.curl_handle = curl_easy_init();
	if (!obj.curl_handle)
		return false;
	
	obj.transfer_size = 0;
	size_t exist_size=0;
	std::filesystem::path localpath(localpath_file);
	if (std::filesystem::exists(localpath))
		exist_size = obj.transfer_size = std::filesystem::file_size(localpath);//已经下载的大小
	
	obj.save_file = fopen(localpath_file.c_str(), "ab+");
	if (!obj.save_file)
	{
		std::cout << "ERROR----fail!!!" << std::endl;
		curl_easy_cleanup(obj.curl_handle);
		return false;
	}
	
	curl_easy_setopt(obj.curl_handle, CURLOPT_RESUME_FROM_LARGE, obj.transfer_size);//续传

	obj.lastruntime = 0;
	curl_easy_setopt(obj.curl_handle, CURLOPT_NOPROGRESS, 0L);
	//curl_easy_setopt(obj.curl_handle, CURLOPT_PROGRESSFUNCTION, xfer_info_func);	
	//curl_easy_setopt(obj.curl_handle, CURLOPT_PROGRESSDATA, (void*) & obj);
	curl_easy_setopt(obj.curl_handle, CURLOPT_XFERINFOFUNCTION, xfer_info_func);	
	curl_easy_setopt(obj.curl_handle, CURLOPT_XFERINFODATA, (void*)&obj);

	curl_easy_setopt(obj.curl_handle, CURLOPT_URL, pageuri.c_str());
	curl_easy_setopt(obj.curl_handle, CURLOPT_USERPWD, "ftp:123456");//ftp用户名和密码
	curl_easy_setopt(obj.curl_handle, CURLOPT_WRITEFUNCTION, recive_data_func);
	curl_easy_setopt(obj.curl_handle, CURLOPT_WRITEDATA, (void*)&obj);
	if (timeout > 0)
	{
		curl_easy_setopt(obj.curl_handle, CURLOPT_CONNECTTIMEOUT, timeout);//连接超时设置
	}
	//下载
	if (curl_easy_perform(obj.curl_handle) != CURLE_OK)
	{
		curl_easy_cleanup(obj.curl_handle);
		fclose(obj.save_file);
		return false;
	}
	else
	{
		curl_easy_cleanup(obj.curl_handle);
		fclose(obj.save_file);
		return obj.transfer_size == (obj.file_size+exist_size);
	}
	return true;
}

7、测试代码

ftp上传和下载

int main()
{
	//测试上传的断点续传
	if (ftp_upload("ftp://127.0.0.1/2021/97.0.4692.71_chrome_installer.exe", 
		"f:\\97.0.4692.71_chrome_installer.exe", 0))
		cout << "UPLOAD OK";
	if (ftp_download("ftp://127.0.0.1/2021/97.0.4692.71_chrome_installer.exe", 
		"f:\\97.0.4692.71_chrome_installer.exe", 0))
		cout << "DOWNLOAD OK";

	return 0;
}

 http下载播放列表和播放内容

int main()
{
	using namespace std;
	TCHAR szPath[MAX_PATH];
	if (!GetModuleFileName(NULL, szPath, MAX_PATH))
	{
		printf("Cannot install service (%d)\n", GetLastError());
		return 0;
	}
	std::filesystem::path current_path(szPath);
	current_path = current_path.parent_path();

	//打开下载文件列表，每行一个文件
	std::ifstream inFile((current_path / "1.txt").c_str(), std::ios::in /*| std::ios::binary*/);
	if (!inFile) 
	{
		std::cout << "error" << std::endl;
		return 0;
	}
	//需要下载的文件列表
	char c[512];
	std::vector<std::string> weburllist;
	while (inFile.getline(c, 511))
	{
		c[511] = 0;
		std::string weburl = c;
		if (weburl.size() > 5)
		{
			if (weburl.find('\r') != -1)
				weburl = weburl.substr(0, weburl.find('\r'));
			weburllist.push_back(weburl);
		}
	}
	inFile.close();

	curl_global_init(CURL_GLOBAL_ALL);

	//逐个下载m3u8文件
	for (auto& pageuri : weburllist)
	{
		cout << "----------------------------------------------------------------------" << endl;
		cout << "web: " << pageuri.c_str() << endl;
		auto index = pageuri.find_last_of('/');
		string urlbase = pageuri.substr(0, index + 1);
		string filename = pageuri.substr(index + 1);
		string name = filename.substr(0,filename.find_last_of('.'));
		string extension = filename.substr(filename.find_last_of('.') + 1);
			
		std::filesystem::path outpath = current_path / name / filename;
		if (!filesystem::exists(current_path / name))
		{
			filesystem::create_directories(current_path / name);
		}

		if (http_download(pageuri, outpath.string()) && extension =="m3u8")
		{
			FILE * save_file = fopen(outpath.string().c_str(), "rb");
			string content;
			char c;
			while (fread(&c, 1, 1, save_file))
			{
				content.push_back(c);
			}					
			fclose(save_file);

			vector<string> downloadlist;//实际文件列表uri
			vector<string> localfilelist;
			for (auto&& i : Split(content, '\n'))
			{
				if (i.length() > 4 && !i.starts_with("#"))
				{
					downloadlist.push_back(urlbase + string(i));					
					localfilelist.push_back((current_path / name / string(i)).string());
				}
			}
			cout << "文件数量： " << downloadlist.size() << endl;
			//下载多个文件
			if (http_multi_download(downloadlist, localfilelist))
			{
			}
			//下载完成
		}
	}
	cout << "download finished , press any key to exit" << endl;
	int value = getchar();
}