libcurl+ncurses 分段range批量下载和进度条显示源码实例

这个例子来自参考文献[1], 那里有很多小bug，我都做了修改，在这里不一一说明了。ncurse界面编程比较容易入门，就是几个接口，网上资料很多，这里不详述了。

//gcc -g mget.c -o mget -lcurl -lncurses -lm
//
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <ncurses.h>
#include <time.h>
#include <curl/curl.h>

WINDOW *win;
int width, height;

WINDOW *create_newwin (int height, int width, int starty, int startx);
void destroy_win (WINDOW * local_win);

typedef struct progress
{
    char byteInterval[64]; //下载分段的字节范围
	int handle; //文件编号或索引
	double totalDownloaded; //累计下载
	double segmentSize;//片段总长字节
	double currentTime;//当前时间戳
	double startTime;//开始时间戳
} *dl_progress;


size_t write_data (void *ptr, size_t size, size_t nmemb, FILE * stream)
{
	size_t written;
	written = fwrite (ptr, size, nmemb, stream);
	return written;
}

static size_t get_size_struct (void *ptr, size_t size, size_t nmemb, void *data)
{
	(void) ptr;
	(void) data;

	// return only the size, dump the rest
	return (size_t) (size * nmemb);
}

int display_progress (dl_progress pgr)
{
	int i = pgr->handle;
    int totalDots = 80; //进度条的长度是80, 剩余的长度是160-80=80
    double totalDownloaded = pgr->totalDownloaded; //已下载片段大小
    time_t diffTime = pgr->currentTime - pgr->startTime;

    double segmentSize = pgr->segmentSize; //总片段大小
    double averageSpeed = pgr->totalDownloaded / diffTime; //平均下载速度
    double fractionDownloaded = totalDownloaded / segmentSize; //下载百分比
    int dots = round (fractionDownloaded * totalDots); //用几个点来表示?

    // create the meter
    mvwprintw (win, i + 3, 2, "%3.0f%% [%2d] [", fractionDownloaded * 100, i);
    int ii = 0;
    for (; ii < dots; ii++)
        mvwprintw (win, i + 3, 13 + ii, "="); //8
    for (; ii < totalDots; ii++)
        mvwprintw (win, i + 3, 13 + ii, " ");
    mvwprintw (win, i + 3, totalDots + 13, "]");//注意间隔是40

    // display some download info
    mvwprintw (win, i + 3, totalDots + 15, "[%s]", pgr->byteInterval);
    mvwprintw (win, i + 3, totalDots + 45, "%03.2f KB/s", averageSpeed / 1024);
    mvwprintw (win, i + 3, width - 18, "%0.2f / %0.2f MB", totalDownloaded / 1024 / 1024, segmentSize / 1024 / 1024);

	wrefresh (win);

	return 0;
}

int progress_func (dl_progress ptr, double totalToDownload, double nowDownloaded, double totalToUpload, double nowUploaded)
{
	time_t seconds;
	seconds = time (NULL);
	ptr->totalDownloaded = nowDownloaded;
	ptr->currentTime = seconds;

	return display_progress (ptr);
}

static double get_download_size (char *url)
{
	CURL *curl;
	CURLcode res;
	double size = 0.0;

	curl = curl_easy_init ();
	curl_easy_setopt (curl, CURLOPT_URL, url);
	curl_easy_setopt (curl, CURLOPT_NOBODY, 1L);
	curl_easy_setopt (curl, CURLOPT_HEADERFUNCTION, get_size_struct);
	curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, 1L);
	res = curl_easy_perform (curl);
	res = curl_easy_getinfo (curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &size);
	if (res != CURLE_OK)
	{
		fprintf (stderr, "curl_easy_getinfo() failed: %s\n", curl_easy_strerror (res));
	}

	curl_easy_cleanup (curl);

	return size;
}

int main (int argc, char *argv[])
{
	if (argc != 3)
	{
		printf ("Incorrect parameters\nUsage: %s <num_parts> <url>\n", argv[0]);
		return -1;
	}

	// setup the start time for average download speed later when we have finished
	time_t startTime;
	startTime = time (NULL);

	// number of parts
	int parts = strtol (argv[1], &argv[1], 10);	// base 10
    struct progress *MyProgress = malloc(sizeof(struct progress) * parts);
    memset(MyProgress, 0 , sizeof(struct progress) * parts);

	// create the window
	initscr ();
	height = parts + 5;
	width = 160; //这里窗口的宽度是160
    int	starty = (LINES - height) / 2;
	int startx = (COLS - width) / 2;
	refresh ();
	win = create_newwin (height, width, starty, startx);

	// setup our vars
	const char *outputfile;
	char *url = argv[2];
	double partSize = 0;
	double segLocation = 0;
	int still_running;
	int i;

	// get file name
	outputfile = strrchr ((const char *) url, '/') + 1;

	// get file size
	double size = get_download_size (argv[2]);
	partSize = size / parts;
    mvwprintw (win, 0, 10, "Downloading %dx%0.2f MB segments (%0.2f MB)  %s", parts, size / parts / 1024 / 1024, size / 1024 / 1024, outputfile);

	// setup curl vars
	FILE *fileparts[parts];
	CURL *single_handles[parts];
	CURLM *multi_handle;
	CURLMsg *msg;
	int msgs_left;

	int error;
	curl_global_init (CURL_GLOBAL_ALL);

	for (i = 0; i < parts; i++)
	{
		time_t seconds;
		seconds = time (NULL);
        memset(MyProgress[i].byteInterval, 0, sizeof(MyProgress[i].byteInterval));
		MyProgress[i].startTime = seconds;
		MyProgress[i].handle = i;
		MyProgress[i].segmentSize = partSize;
		MyProgress[i].totalDownloaded = 0;

		// setup our output filename
		char filename[50] = { 0 };
		snprintf (filename, sizeof (filename), "%s.part.%0d", outputfile, i);

		// allocate curl handle for each segment
		single_handles[i] = curl_easy_init ();
		fileparts[i] = fopen (filename, "w");

		double nextPart = 0;
		if (i == parts - 1)
		{
			nextPart = size;
		}
		else
		{
			nextPart = segLocation + partSize - 1;
		}

		char range[64] = { 0 };
		snprintf (range, sizeof (range), "%12.0f-%12.0f", segLocation, nextPart);
        memcpy(MyProgress[i].byteInterval, range, strlen(range));

		// set some curl options.
		curl_easy_setopt (single_handles[i], CURLOPT_URL, url);
		curl_easy_setopt (single_handles[i], CURLOPT_RANGE, range);  //设置range请求
		curl_easy_setopt (single_handles[i], CURLOPT_FOLLOWLOCATION, 1L);
		curl_easy_setopt (single_handles[i], CURLOPT_WRITEFUNCTION, write_data);//设置接收到文件内容回调
		curl_easy_setopt (single_handles[i], CURLOPT_WRITEDATA, fileparts[i]); //写数据回调的最后一个参数
		curl_easy_setopt (single_handles[i], CURLOPT_NOPROGRESS, 0); //设置进度回调功能
		curl_easy_setopt (single_handles[i], CURLOPT_PROGRESSFUNCTION, progress_func); //设置进度回调函数
		curl_easy_setopt (single_handles[i], CURLOPT_PROGRESSDATA, &MyProgress[i]); //设置进度回调函数的第一个参数

		segLocation += partSize;
	}

	multi_handle = curl_multi_init ();

	// add all individual transfers to the stack
	for (i = 0; i < parts; i++)
	{
		curl_multi_add_handle (multi_handle, single_handles[i]);
	}

	curl_multi_perform (multi_handle, &still_running);

	do
	{
		struct timeval timeout;
		int rc;					// return code
		fd_set fdread;
		fd_set fdwrite;
		fd_set fdexcep;			// file descriptor exception
		int maxfd = -1;

		long curl_timeo = -1;

		FD_ZERO (&fdread);
		FD_ZERO (&fdwrite);
		FD_ZERO (&fdexcep);

		// set a suitable timeout to play with
		timeout.tv_sec = 100 * 1000;
		timeout.tv_usec = 0;

		curl_multi_timeout (multi_handle, &curl_timeo);
		if (curl_timeo >= 0)
		{
			timeout.tv_sec = curl_timeo / 1000;
			if (timeout.tv_sec > 1)
			{
				timeout.tv_sec = 1;
			}
			else
			{
				timeout.tv_usec = (curl_timeo % 1000) * 1000;
			}
		}

		curl_multi_fdset (multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);

		rc = select (maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);

		switch (rc)
		{
		case -1:
			fprintf (stderr, "Could not select the error\n");
			break;
		case 0:
			/* timeout */
		default:
			curl_multi_perform (multi_handle, &still_running);
			break;
		}
	}
	while (still_running);

	while ((msg = curl_multi_info_read (multi_handle, &msgs_left)))
	{
		if (msg->msg == CURLMSG_DONE)
		{
			int index, found = 0;

			for (index = 0; index < parts; index++)
			{
				found = (msg->easy_handle == single_handles[index]);
				if (found)
					break;
			}
		}
	}

	// clean up our multi handle
	curl_multi_cleanup (multi_handle);

	// free up the curl handles
	for (i = 0; i < parts; i++)
	{
		curl_easy_cleanup (single_handles[i]);
	}

	// close ncurses window
    destroy_win(win);
	endwin ();
    free(MyProgress);

	// send some output to the console for records sake.
	time_t endTime;
	endTime = time (NULL);
	printf ("Downloaded %0.2f MB in %ld seconds\n", partSize * parts / 1024 / 1024, endTime - startTime);
	printf ("%0.2f KB/s (average)\n", (partSize * parts / (endTime - startTime) / 1024));

	return 0;
}

WINDOW *create_newwin (int height, int width, int starty, int startx)
{
	WINDOW *local_win;

	local_win = newwin (height, width, starty, startx);
	box (local_win, 0, 0);
	wrefresh (local_win);

	return local_win;
}

void destroy_win (WINDOW * local_win)
{
	/* box(local_win, ' ', ' '); : This won't produce the desired
	 * result of erasing the window. It will leave it's four corners
	 * and so an ugly remnant of window.
	 */
	wborder (local_win, '|', '|', '-', '-', '+', '+', '+', '+');
	/* The parameters taken are
	 * 1. win: the window on which to operate
	 * 2. ls: character to be used for the left side of the window
	 * 3. rs: character to be used for the right side of the window
	 * 4. ts: character to be used for the top side of the window
	 * 5. bs: character to be used for the bottom side of the window
	 * 6. tl: character to be used for the top left corner of the window
	 * 7. tr: character to be used for the top right corner of the window
	 * 8. bl: character to be used for the bottom left corner of the window
	 * 9. br: character to be used for the bottom right corner of the window
	 */
	wrefresh (local_win);
	delwin (local_win);
}

运行方法

需要两个参数，第一个参数是，分段个数，第二个参数是大文件的链接。默认是大文件下载，上G的文件，可以分为几十兆一个分段，并行异步下载（注意是单线程异步批量，不是多线程），效果还可以。

./mget 20 "http://cdimage.ubuntu.com/releases/14.04/release/ubuntu-14.04-desktop-amd64+mac.iso"
Downloaded 962.00 MB in 867 seconds
1136.20 KB/s (average)

运行截图

附加说明

设计UI界面是个费时费力的细致活, 需要根据屏幕分辨率设计好. 我的ThinkPad是720P的屏，当前的效果给出的是80宽度的框, 进度条使用"="表示, 宽度是40, 高度是文件个数,加上5行空白, 上面是3, 下面是2.一般的笔记本通常没有高的分辨率，建议框的宽度是40就可以，其他的调整按照参考文献[1]设置就够用了。

待解决的问题

如何将这几个文件的即时速度的和统计出来?我暂时没有找到较好的方法.希望大牛指点.

参考文献

[1].https://github.com/logikaljay/mget