linux 下载web服务器文件

一、linux C下发送http请求，下载web服务器文件

注：发生的请求为http开头的web服务器，数据没有经过加密传输， 请求https 服务器上文件，需要openssl的接口来认证密钥，才能请求文件。

1、封装的http请求接口

#include <stdio.h>

#include <string.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include <unistd.h>
#include <netdb.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <pthread.h>
#include <sys/time.h>


typedef struct {    
int client_socket;
int quit;
int status_code;//HTTP/1.1 '200' OK    
char content_type[128];//Content-Type: application/gzip    
long content_length;//Content-Length: 11683079    
}RespHead_t;
static RespHead_t resp;


static void parse_url(const char *url, char *domain, int *port, char *file_name)
{  
/*通过url解析出域名, 端口, 以及文件名*/
int j = 0;  
int start = 0;
*port = 80;
char *patterns[] = {"http://", "https://", NULL};
int i;
for(i = 0; patterns[i]; i++)
if (strncmp(url, patterns[i], strlen(patterns[i])) == 0)
start = strlen(patterns[i]);
//解析域名, 这里处理时域名后面的端口号会保留
for( i = start; url[i] != '/' && url[i] != '\0'; i++, j++)
domain[j] = url[i];
domain[j] = '\0';
//解析端口号, 如果没有, 那么设置端口为80
char *pos = strstr(domain, ":");
  if (pos)        
sscanf(pos, ":%d", port);
//删除域名端口号
for( i = 0; i < (int)strlen(domain); i++)
    {        
    if (domain[i] == ':')        
{            
domain[i] = '\0';
break;
}
}
    //获取下载文件名    
    j = 0;    
for ( i = start; url[i] != '\0'; i++)
{        
if (url[i] == '/')        
{            
if (i !=  strlen(url) - 1)                
j = 0;            
continue;        
}        
else            
file_name[j++] = url[i];    
}    
file_name[j] = '\0';
}


static void get_resp_header(RespHead_t *resp,const char *response)
{    
/*获取响应头的信息*/    
   
char *pos = strstr(response, "HTTP/");    
if (pos)        
sscanf(pos, "%*s %d", &resp->status_code);//返回状态码    
pos = strstr(response, "Content-Type:");//返回内容类型    
if (pos)        
sscanf(pos, "%*s %s", resp->content_type);    
pos = strstr(response, "Content-Length:");//内容的长度(字节)    
if (pos)        
sscanf(pos, "%*s %ld", &resp->content_length);    
}
static void get_ip_addr(char *domain, char *ip_addr)
{    
/*通过域名得到相应的ip地址*/    
struct hostent *host = gethostbyname(domain);    
int i;    
if (!host)    
{        
ip_addr = NULL;        
return;    
}    
for (i = 0; host->h_addr_list[i]; i++)    
{        
strcpy(ip_addr, inet_ntoa( * (struct in_addr*) host->h_addr_list[i]));        
break;    
}
}
static void progressBar(long cur_size, long total_size)
{    
/*用于显示下载进度条*/    
float percent = (float) 
cur_size / total_size;    
const int numTotal = 50;    
int numShow = (int)(numTotal * percent);    
if (numShow == 0)        
numShow = 1;    
if (numShow > numTotal)        
numShow = numTotal;    
char sign[51] = {0};    
memset(sign, '=', numTotal);    
printf("\r%.2f%%\t[%-*.*s] %.2f/%.2fMB", percent * 100, numTotal, numShow, sign, cur_size / 1024.0 / 1024.0, total_size / 1024.0 / 1024.0);    fflush(stdout);    
if (numShow == numTotal)        
printf("\n");
}

void quitDownhttpFile(void)
{
int timeout=0;
resp.quit=1;
if(resp.client_socket>0){
close(resp.client_socket);
resp.client_socket=-1;
}
do{
if(++timeout>20)
break;
usleep(100);
}while(resp.quit!=1);
}
int DownhttpFile(const char *url,void StartDownFile(const char *filename,int streamLen),void GetStreamData(const char *data,int size),void EndDownFile(int FileSize))
{
char domain[64] = {0}; 
char ip_addr[16] = {0};    
int port = 80; 
char file_name[256] = {0}; 
  int ret=0;
printf(" DownhttpFile  url=%s\n",url);   


memset(&resp,0,sizeof(RespHead_t));
resp.quit=0;
printf("1: Parsing url... \n"); 
parse_url(url, domain, &port, file_name);
printf("2: Get ip address...\n"); 
printf("\n>>>>Detail<<<<"); 
printf("URL: %s\n", url); 
printf("DOMAIN: %s\n", domain);    
printf("PORT: %d\n", port);    
printf("FILENAME: %s\n\n", file_name);
get_ip_addr(domain, ip_addr); 
if (strlen(ip_addr) == 0) 
{  
printf("can not get ip address\n");   
return -1;  
}
printf("IP: %s\n", ip_addr);
//设置http请求头信息
char header[1024] = {0};
sprintf(header,"GET %s HTTP/1.1\r\n"\
"Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n"\
"User-Agent:Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537(KHTML, like Gecko) Chrome/47.0.2526Safari/537.36\r\n"\ 
"Host:%s\r\n"\ 
"Connection:close\r\n"\   
"\r\n"\   
,url, domain); 
//printf("%s\n%d", header, (int) strlen(header)); 
//创建套接字
resp.client_socket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); 
if (resp.client_socket < 0) 
{  
printf("invalid socket descriptor: %d\n", resp.client_socket); 
return -1;
}  
//创建地址结构体  
struct sockaddr_in addr;
memset(&addr, 0, sizeof(addr));    
addr.sin_family = AF_INET; 
addr.sin_addr.s_addr = inet_addr(ip_addr); 
addr.sin_port = htons(port);//连接服务器
printf("3: Connect server...\n"); 
int res = connect(resp.client_socket, (struct sockaddr *) &addr, sizeof(addr));
if (res == -1) 
{
printf("connect failed, return: %d\n", res);
ret=-1;
goto exit0;
}
printf("4: Send request...\n");
//向服务器发送下载请求
write(resp.client_socket, header, strlen(header));
int mem_size = 4096;
int length = 0;
int len;
char *buf = (char *) calloc(1,mem_size * sizeof(char)); 
if(buf==NULL)
{
ret=-1;
goto exit0;
}
char *response = (char *) malloc(mem_size * sizeof(char)); 
if(response==NULL)
{
ret=-1;
goto exit1;
}
char * temp =NULL;
//每次单个字符读取响应头信息, 仅仅读取的是响应部分的头部, 后面单独开线程下载

while ((len = read(resp.client_socket, buf, 1)) != 0)
{
if(length + len > mem_size)
{  
//动态内存申请, 因为无法确定响应头内容长度
printf("length + len =%d mem_size=%d\n",length + len,mem_size);
mem_size *= 2; 
temp= (char *) realloc(response, sizeof(char) * mem_size);
if (temp == NULL) 
{  
printf("realloc failed\n");
ret=-1;
goto exit2;
}
response = temp;
}
buf[len] = '\0';
sprintf(response+length,"%s",buf);
//找到响应头的头部信息, 两个"\n\r"为分割点 
int flag = 0;
int i;
for (i = strlen(response) - 1; response[i] == '\n' || response[i] == '\r'; i--, flag++);
if (flag == 4) 
break;  
length += len; 
}  
//printf("\n>>>>Response header:<<<<\n%s", response);
get_resp_header(&resp,response);
StartDownFile((const char *)file_name,resp.content_length);
printf("5: Start thread to download...\n");  
int buf_len = mem_size;//read 4k each time    

length = 0;    
while ((len = read(resp.client_socket, buf, buf_len)) != 0 && length < resp.content_length)    
{
if(resp.quit==1){
printf("GetStreamData quit ...\n");
break;
}
GetStreamData(buf,len);
length += len;        
//progressBar(length, resp.content_length);
}    
if (length == resp.content_length)        
{
printf("Download successful ^_^\n\n");
}
EndDownFile(resp.content_length);
exit2:
free(response);
exit1:
free(buf);
exit0:
if(resp.client_socket>0)
close(resp.client_socket);
resp.client_socket=-1;
resp.quit=2;
return ret;
}


static FILE *fp=NULL ;
static void StartDownFile(const char *filename,int fileSize)
{
fp= fopen(filename, "w+");    
if (fp ==NULL)    
{        
printf("Create file failed\n");   
}  
}
static void GetStreamData(const char *data,int size)
{
  fwrite( data, size,1fp);
}
static void EndDownFile(int FileSize)
{
fclose(fp);

}

int main(int argc, char  *argv[])
{    
char url[]="http://fdfs.xmcdn.com/group7/M01/A3/8D/wKgDX1d2Rr6w3CegABHDHZzUiUs448.mp3";
int i=0;
DownhttpFile((const char *)url,StartDownFile,GetStreamData,EndDownFile);
return 0;
}

2、经过验证可以下载web服务上的文件，但换了多个网络测试之后，发现部分网络发送http请求之后，服务器回出现302的响应。

重定向这个请求到另外一个服务器，导致下载不了文件。为了稳定性，最后还是采用 libghttp这个库，来请求web服务器文件