1. 软件介绍
qdown 是一款开源的 HTTP 多线程下载软件。
特点：多线程，支持服务器重定向，支持断点续传。
平台： Solaris/FreeBSD/Windows(CygWin)
2. 如何使用
usage: qdown URL [thread_amount] [save as]
example: qdown http://www.baidu.com/img/logo.gif 5 /home/sunjoy/log.gif
3. 如何编译
On Solaris: cc -lsocket -lnsl qdown.c
On FreeBSD: gcc -pthread qdown.c
或者用 sunstudio 打开工程文件编译
4. 基本原理
4.1 多线程原理
HTTP 协议规定在请求报头中加入 Range: bytes=%d-%d (%d 代表整数 ) 来下载指定范围的块儿，
因此根据文件的总大小， qdown 开启多个线程分别下载各个部分，最终完成下载整个文件。
4.2 服务器重定向
很多情况下，当客户端发起 GET 请求后，服务器可能通过 Location: xxxxx 来告诉客户端重定向
到新的 URL ，当 qdown 遇到这种情况时会去下载新的 URL 指定的文件。 qdown 最多允许 5 次重定向。
4.3 断点续传
由于程序被中断或者网络故障等原因可能导致一个文件没有下载完全。 qdown 在下载过程中会
维护一个 .cfg 文件来记录个线程的下载情况，当重新下载时， qdown 会根据 .cfg 文件的记录从
上次断掉的地方开始下载。
5. 改进方向
预计在下一版本中加入对 FTP URL 的支持
/**
** description:qdown is a multithread downloader
** author:Sunjoy
** email:fxsjy @@@ yahoo.com.cn
** from:ICT.CAS.
** date:2007-9-10
**
*/
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <pthread.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <netdb.h>
#define MAX_THREAD 100
typedef struct URLInfo
{
char schema[8];
char host[256];
char host_name[256];
unsigned int port;
char file[256];
}URLInfo;
typedef struct Connection
{
int sock;
URLInfo url_info;
int avaliable;
}Connection;
typedef struct Resource
{
char file_url[256];
int file_size;
char file_name[256];
}Resource;
typedef struct ThreadArg
{
Resource* res;
int start_pos;
int limit;
int no;
}ThreadArg;
typedef struct BreakPoint
{
int downloaded;
int thread_amount;
int tasks[MAX_THREAD][2];
}BreakPoint;
pthread_mutex_t g_mut;
int g_total=0;
int g_downloaded=0;
BreakPoint g_breakpoint;
URLInfo parse_url(const char *url);
Connection open_url(const char * url);
Resource get_resource(const char *url);
void join_url(const char* old_url,const char* redirect,char * new_url);
void download(const char* url,int thread_amount,const char* file_name);
void* download_part(void* args);
void* monitor(void *args);
void store_breakpoint(char * cfgName);
void store_breakpoint(char * cfgName)
{
int z;
FILE* f;
f=fopen(cfgName,"w");
fprintf(f,"%d/n",g_breakpoint.downloaded);
fprintf(f,"%d/n",g_breakpoint.thread_amount);
for(z=0;z<g_breakpoint.thread_amount;z++){
fprintf(f,"%d-%d/n",g_breakpoint.tasks[z][0],g_breakpoint.tasks[z][1]);
}
fclose(f);
}
void join_url(const char* old_url,const char* redirect,char * new_url)
{
char stack1[256][256]={0},stack2[256][256]={0};
int i=0,j=0,p1=0,p2=0;
char seg[256]={0};
URLInfo temp_urlinfo;
memset(new_url,0,sizeof(new_url));
if(strstr(redirect,"://")!=NULL){
strcpy(new_url,redirect);
}
else{
while(1){
while(redirect[i]!='/' && redirect[i]!=0){
seg[j++]=redirect[i++];
}
strcpy(stack1[p1++],seg);
memset(seg,0,sizeof(seg));
j=0;
if(redirect[i]==0)
break;
i++;
}
for(i=0;i<p1;i++){
if(!strcmp(stack1[i],"..") && p2>-1)
p2--;
else if(strcmp(stack1[i],".")){
strcpy(stack2[p2++],stack1[i]);
}
}
//printf("##%s/n",stack2[0]);
if(!strcmp(stack2[0],"")){
temp_urlinfo=parse_url(old_url);
sprintf(new_url,"%s://%s:%d/",temp_urlinfo.schema,temp_urlinfo.host,temp_urlinfo.port);
}
else{
i=strlen(old_url)-1;
while(old_url[i]!='/')
i--;
//printf("##%c/n",old_url[i]);
strncpy(new_url,old_url,i+1);
new_url[i+1]=0;
}
//printf("##%s/n",new_url);
for(j=0;j<p2-1;j++){
strcat(new_url,stack2[j]);
strcat(new_url,"/");
}
strcat(new_url,stack2[p2-1]);
}
}
URLInfo parse_url(const char* url){
int i=0,j=0;
char schema[8]={0};
char host[256]={0};
char port[8]={0};
char file[256]={0};
char IP[32]={0};
URLInfo url_info;
struct hostent* hptr;
while(url[i]!=':'){
schema[j++]=url[i++];
}
for(i+=3,j=0;url[i]!=':' && url[i]!='/' && url[i]!=0;){
host[j++]=url[i++];
}
if(url[i]==':'){
for(i+=1,j=0;url[i]!='/';){
port[j++]=url[i++];
}
sscanf(port,"%d",&url_info.port);
}
else{
url_info.port=80;
}
if(url[i]!=0){
for(j=0;url[i]!=0;){
file[j++]=url[i++];
}
}
else{
file[0]='/';
}
strcpy(url_info.schema,schema);
strcpy(url_info.file,file);
strcpy(url_info.host_name,host);
hptr=gethostbyname(host);
if(hptr!=NULL){
strcpy(url_info.host,
inet_ntop(hptr->h_addrtype,*(hptr->h_addr_list),IP,sizeof(IP))
);
}
//printf("%s/n",url_info.host);
return url_info;
}
Connection open_url(const char* url){
Connection conn;
struct sockaddr_in remote_addr,local_addr;
conn.avaliable=0;
conn.url_info=parse_url(url);
local_addr.sin_family=AF_INET;
local_addr.sin_addr.s_addr=htonl(INADDR_ANY);
local_addr.sin_port=htons(0);
remote_addr.sin_family=AF_INET;
remote_addr.sin_addr.s_addr=inet_addr(conn.url_info.host);
remote_addr.sin_port=htons(conn.url_info.port);
conn.sock=socket(AF_INET,SOCK_STREAM,0);
if(bind(conn.sock,
(struct sockaddr*)&local_addr,
sizeof(local_addr))<0){
printf("bind error/n");
}
if(conn.sock){
if(
connect(conn.sock,(struct sockaddr*)&remote_addr,sizeof(remote_addr))!=-1
){
conn.avaliable=1;
}
}
return conn;
}
Resource get_resource(const char* url){
char pack[1024]={0};
char buf[1024]={0};
char redirect[256]={0},new_url[256]={0},old_url[256]={0};
static int redirect_count=0;
char* i;
char* j;
char* z;
Resource res;
Connection conn=open_url(url);
if(!conn.avaliable){
return res;
}
sprintf(pack,"GET %s HTTP/1.1/nHost: %s/nAccept: */*/nReferer: http://%s/nUser-Agent: Mozilla/4.0 (compatible; MSIE 5.00; Windows 98)/nPragma: no-cache/nCache-Control: no-cache/nConnection: close/n/n",conn.url_info.file,conn.url_info.host_name,conn.url_info.host_name);
send(conn.sock,pack,strlen(pack),0);
recv(conn.sock,buf,sizeof(buf),0);
//printf("%s/n",buf);
if(strstr(buf,"HTTP/1.1 404")!=NULL || strstr(buf,"HTTP/1.0 404")!=NULL){
return res;
}
i=(char *)strstr(buf,"Location:");
if(i!=NULL && redirect_count<5){
sscanf(i,"Location: %s",redirect);
sprintf(old_url,"%s://%s:%d%s",conn.url_info.schema,conn.url_info.host_name,conn.url_info.port,conn.url_info.file);
join_url(old_url,redirect,new_url);
//printf("@#%s/n",new_url);
redirect_count++;
return get_resource(new_url);
}
i=(char *)strstr(buf,"Content-Length:");
if(i!=NULL){
sscanf(i,"Content-Length: %d",&res.file_size);
}
strcpy(res.file_url,url);
//printf("#%d/n",res.file_size);
for(z=(char*)url;(j=strstr(z,"/"))!=NULL;){
z=j+sizeof(char);
}
strcpy(res.file_name,z);
close(conn.sock);
return res;
}
void* download_part(void * args)
{
ThreadArg* targ=(ThreadArg*)args;
Connection conn;
FILE* f=NULL;
char pack[1024]={0};
char buf[1024]={0};
int i=0,ct=0;
char* body=NULL;
//printf("%s,%d-%d/n",targ->res->file_url, targ->start_pos,targ->limit);
conn=open_url(targ->res->file_url);
while(!conn.avaliable){
sleep(1);
conn=open_url(targ->res->file_url);
}
if(conn.avaliable){
f=fopen(targ->res->file_name,"rb+");
fseek(f,targ->start_pos,0);
sprintf(pack,"GET %s HTTP/1.1/nHost: %s/nAccept: */*/nReferer: http://%s/nUser-Agent: Mozilla/4.0 (compatible; MSIE 5.00; Windows 98)/nRange: bytes=%d-%d/nPragma: no-cache/nCache-Control: no-cache/n/n",conn.url_info.file,conn.url_info.host_name,conn.url_info.host_name,targ->start_pos,targ->start_pos+targ->limit-1);
//printf("%s",pack);
begin_down:
send(conn.sock,pack,strlen(pack),0);
i=recv(conn.sock,buf,sizeof(buf),0);
if(strstr(buf,"HTTP/1.1 206")==NULL && strstr(buf,"HTTP/1.0 206")==NULL && strstr(buf,"HTTP/1.1 200")==NULL && strstr(buf,"HTTP/1.0 200")==NULL){
sleep(2);
memset(buf,0,sizeof(buf));
conn=open_url(targ->res->file_url);
goto begin_down;
}
//printf("##%s/n",body);
body=strstr(buf,"/r/n/r/n")+4;
if(body!=NULL){
i=i-(body-buf);
fwrite(body,sizeof(char),i,f);
//printf("@@@@%x/n",buf);
fflush(f);
ct+=i;
pthread_mutex_lock(&g_mut);
g_downloaded+=i;
pthread_mutex_unlock(&g_mut);
while(ct< targ->limit){
i=recv(conn.sock,buf,sizeof(buf),0);
if(i==0){
fclose(f);
conn.avaliable=0;
while(!conn.avaliable){
sleep(2);
//printf("waiting.../n");
conn=open_url(targ->res->file_url);
}
memset(pack,0,sizeof(pack));
memset(buf,0,sizeof(buf));
sprintf(pack,"GET %s HTTP/1.1/nHost: %s/nAccept: */*/nReferer: http://%s/nUser-Agent: Mozilla/4.0 (compatible; MSIE 5.00; Windows 98)/nRange: bytes=%d-%d/nPragma: no-cache/nCache-Control: no-cache/n/n",conn.url_info.file,conn.url_info.host_name,conn.url_info.host_name,targ->start_pos+ct,targ->start_pos+targ->limit-1);
f=fopen(targ->res->file_name,"rb+");
fseek(f,targ->start_pos+ct,0);
goto begin_down;
}
fwrite(buf,sizeof(char),i,f);
fflush(f);
ct+=i;
pthread_mutex_lock(&g_mut);
g_downloaded+=i;
g_breakpoint.tasks[targ->no][0]=targ->start_pos+ct;
g_breakpoint.tasks[targ->no][1]=targ->limit-ct;
g_breakpoint.downloaded=g_downloaded;
pthread_mutex_unlock(&g_mut);
}
fclose(f);
g_breakpoint.downloaded=g_downloaded;
close(conn.sock);
}
}
pthread_exit(NULL);
}
void* monitor(void* args){
float p;
int i,j,z,old;
FILE* f;
char cfgName[256];
strcpy(cfgName,(char*)args);
strcat(cfgName,".cfg");
while(1){
p=g_downloaded/(g_total+0.0);
if(g_downloaded>=g_total)
break;
i=p*100/10;
if(old!=g_downloaded){
printf("/r");
for(j=0;j<i;j++){
printf("==");
}
printf("%2.0f%%",p*100);
fflush(stdout);
store_breakpoint(cfgName);
old=g_downloaded;
}
}
printf("/r====================100%%/n");
remove(cfgName);
pthread_exit(NULL);
}
void download(const char* url,int thread_amount,const char* file_name)
{
ThreadArg targs[MAX_THREAD];
pthread_attr_t * thAttr = NULL;
pthread_t tids[MAX_THREAD],monitor_id,controler_id;
Resource res;
int i,block_size,t_start_pos,t_limit;
FILE* f;
char cfgName[256]={0};
if(thread_amount>MAX_THREAD)
return;
res=get_resource(url);
if(!strcmp(res.file_url,""))
return;
if(strcmp(file_name,""))
strcpy(res.file_name,file_name);
if(!strcmp(res.file_name,""))
strcpy(res.file_name,"default_down");
if(res.file_size<1000000)
thread_amount=1;
block_size=res.file_size/thread_amount;
pthread_mutex_init(&g_mut,NULL);
strcpy(cfgName,res.file_name);
strcat(cfgName,".cfg");
printf("downloading %s,%d bytes... /n",res.file_name,res.file_size);
if(fopen(cfgName,"r")==NULL){
new_task:
f=fopen(res.file_name,"wb");
if(f==NULL){
strcpy(res.file_name,"default_down");
f=fopen(res.file_name,"wb");
}
fclose(f);
g_total=res.file_size;
for(i=0;i<thread_amount;i++){
targs[i].res=&res;
targs[i].start_pos=block_size*i;
targs[i].limit=block_size;
if(i==thread_amount-1)
targs[i].limit+= (res.file_size%thread_amount);
targs[i].no=i;
g_breakpoint.tasks[i][0]=targs[i].start_pos;
g_breakpoint.tasks[i][1]=block_size;
pthread_create(&tids[i], thAttr, download_part, (void *)&targs[i]);
}
}
else{
f=fopen(cfgName,"r");
if(fscanf(f,"%d",&g_downloaded)==-1)
goto new_task;
//printf("#%d/n",g_downloaded);
g_total=res.file_size;
fscanf(f,"%d",&thread_amount);
for(i=0;i<thread_amount;i++){
fscanf(f,"%d-%d",&t_start_pos,&t_limit);
targs[i].res=&res;
targs[i].start_pos=t_start_pos;
targs[i].limit=t_limit;
targs[i].no=i;
g_breakpoint.tasks[i][0]=targs[i].start_pos;
g_breakpoint.tasks[i][1]=t_limit;
pthread_create(&tids[i], thAttr, download_part, (void *)&targs[i]);
}
fclose(f);
}
pthread_create(&monitor_id,NULL,monitor,(void *)res.file_name);
g_breakpoint.thread_amount=thread_amount;
g_breakpoint.downloaded=g_downloaded;
//printf("#%d/n",g_downloaded);
/*for(i=0;i<thread_amount;i++){
pthread_join(tids[i],NULL);
}*/
pthread_join(monitor_id,NULL);
}
//-------------------------------- MAIN--------------------------------//
int main (int ac, char * av[])
{
int thread_amount=5;
char file_name[256]={0};
if(ac<2){
printf("usage: qdown URL [thread_amount] [save as]/n");
printf("example: qdown http://www.baidu.com/img/logo.gif 5 /home/sunjoy/log.gif/n");
}
else{
if(ac>=3)
sscanf(av[2],"%d",&thread_amount);
if(ac>=4){
strcpy(file_name,av[3]);
}
download(av[1],thread_amount,file_name);
}
return 0;
}
369
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
