一个分块上传/下载文件的线程模型-生产消费者模式 C/C++

一个分块上传/下载文件的线程模型-生产消费者模式。

当上传一个大文件时，需要将文件分块上传。这里将文件做为一个node，然后分割成若干块任务task（具体分割多少块取决于每块的大小，一般建议将文件以10M每块分割），将task放入任务列表g_task_list。

再建立几个工作线程woker_thread，工作线程去取task，取到并完成任务后remove_upload_task，继续取下一个任务，取不到就等待。

每个task的具体工作是：

如果是上传，可以直接fopen，然后fseek到对应的位置读取数据，再上传；多个task，就多个fopen,由于读取的位置不同，无需再添加文件锁。

如果是下载，task可以分别保存成不同的小文件，全部下载完毕后，再合成一个文件。

代码如下，在windows上已经测试OK,Linux请自动测试，windows上也有pthread库，请自动搜索添加。

#include <iostream>
#include <stdlib.h>
#include <pthread.h>
#include <list>
#include <stdarg.h>
#include <windows.h>

#define P printf
#define PINFO printf

enum upload_task_state_st {
	UPLOAD_WAITING = 0,
	UPLOAD_RUNNING = 0x01,
	UPLOAD_PAUSE = 0x02,
	UPLOAD_FINISH = 0x03,
	UPLOAD_ERROR = 0x04
};

typedef struct upload_file_node_ {
	char file_full_path[_MAX_PATH];
	long long file_size;
	int max_running_thread;
	std::list<int> sections;
}upload_file_node;

typedef struct upload_task_ {
	int task_state;//upload_task_state_st
	int index;
	upload_file_node *node;
}upload_task;

std::list<upload_task*> g_task_list;
static pthread_mutex_t g_upload_thread_mutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t g_upload_thread_cond = PTHREAD_COND_INITIALIZER;

static upload_task *get_upload_task()
{
	upload_task* task = NULL;
	for (std::list<upload_task*>::iterator it = g_task_list.begin(); it != g_task_list.end(); ++it) {
		if (UPLOAD_WAITING == (*it)->task_state) {
			task = (*it);
			task->task_state = UPLOAD_RUNNING;
			break;
		}
	}
	return task;
}

static void push_upload_task(upload_file_node* node)
{
	int i;
	std::list<int>::iterator it = node->sections.begin();

	pthread_mutex_lock(&g_upload_thread_mutex);
	for (i = 0; i < node->sections.size(); i++) {
		upload_task *task = (upload_task*)calloc(1, sizeof(upload_task));
		task->node = node;
		task->index = (*it);
		++it;
		task->task_state = UPLOAD_WAITING;
		g_task_list.push_back(task);
	}
	pthread_cond_broadcast(&g_upload_thread_cond);
	pthread_mutex_unlock(&g_upload_thread_mutex);
}

static void remove_upload_task(upload_task* task)
{
	pthread_mutex_lock(&g_upload_thread_mutex);
	for (std::list<upload_task*>::iterator it = g_task_list.begin(); it != g_task_list.end(); ++it) {
		if (task == *it) {
			P("remove index=%d\n", task->index);
			task->node->sections.remove(task->index);
			g_task_list.remove(task);
			free(task);
			break;
		}
	}
	pthread_mutex_unlock(&g_upload_thread_mutex);
}

void sleepcp(int milliseconds) 
{
#ifdef _WIN32
	Sleep(milliseconds);
#else
	usleep(milliseconds * 1000);
#endif 
}

static void *woker_thread(void *arg)
{
	(void)arg;
	upload_task* task = NULL;

	while (1) {
		pthread_mutex_lock(&g_upload_thread_mutex);
		while (NULL == (task = get_upload_task())) {
			P("%s","sleeping\n");
			pthread_cond_wait(&g_upload_thread_cond, &g_upload_thread_mutex);
		}
		pthread_mutex_unlock(&g_upload_thread_mutex);
		P("doing %s index=%d\n", task->node->file_full_path,task->index);
		sleepcp(100);
		remove_upload_task(task);
	}

	return NULL;
}
#define THREAD_NUM 3
void test_pthread()
{
	int i,j;
	pthread_t tids[THREAD_NUM];

	for (i = 0; i < THREAD_NUM; i++) {
		pthread_create(&tids[i], NULL, woker_thread, NULL);
	}

	for (j = 0; j < 3; j++) {
		upload_file_node node;
		snprintf(node.file_full_path,sizeof(node.file_full_path),"File_%d",j);
		node.file_size = 1213;
		node.max_running_thread = THREAD_NUM;

		for (i = 0; i < 20; i++) {
			node.sections.push_back(i);
		}
		push_upload_task(&node);

AGAIN:
		pthread_mutex_lock(&g_upload_thread_mutex);
		if (0 != node.sections.size()) {
			PINFO("there still have %d task\n", node.sections.size());
			pthread_cond_broadcast(&g_upload_thread_cond);
			pthread_mutex_unlock(&g_upload_thread_mutex);
			sleepcp(1000);
			goto AGAIN;
		}	
		pthread_mutex_unlock(&g_upload_thread_mutex);
	}
	
	P("%s","all task has finished!\n");
	
	for (i = 0; i < THREAD_NUM; i++) {
		pthread_join(tids[i],NULL);
	}

}

int main()
{
	test_pthread();
    getchar();
	return 0;
}