C语言的简单爬虫（个人学习笔记）

实现代码：

#include <WinSock2.h>
#include <ws2tcpip.h>

#include<string.h>
#include <stdio.h>
#include <stdlib.h>
#include <windows.h>
#pragma warning(disable:4996)
#pragma comment(lib,"ws2_32.lib")

void check_result(bool unsuccessful, const char* msg)
{
    if (unsuccessful)
    {
        printf("error:%s\n", msg);
        system("pause"); exit(0);

    }

}

//域名解析
//参数
//返回数据
//编码转换 utf8->gb2312

//解析域名，ping找出ip，访问网站，返回数据转码，匹配查询

WSADATA WsaData{};
//获取IP
char* get_ip(char* url)
{
    WSAStartup(MAKEWORD(2, 2), &WsaData);
    hostent* phost = gethostbyname(url);
    in_addr addr;
    char* ip = 0;
    char* p = phost->h_addr_list[0];
    memcpy(&addr.S_un.S_addr, p, phost->h_length);
    ip = inet_ntoa(addr);
    strcat(ip, "\0");
    return ip;
}

//网站访问
char* access_url(char* url, char* ip, char* content)
{
    //初始化网络环境
    auto result = WSAStartup(MAKEWORD(2, 2), &WsaData);
    check_result(result != 0, "网络初始化失败");
    check_result(WsaData.wVersion != MAKEWORD(2, 2), "无法请求指定版本的模块");

    //创建套接字
    auto client = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
    check_result(client == INVALID_SOCKET, "网络环境初始化失败2");

    //绑定套接字
    sockaddr_in client_address{ AF_INET, htons(80) };
    inet_pton(AF_INET, ip, &client_address.sin_addr);
    result = connect(client, (sockaddr*)&client_address, sizeof(sockaddr_in));
    check_result(client == SOCKET_ERROR, "绑定套接字失败");

    //请求头拼接
    char* splicing = (char*)malloc(0x100);
    memset(splicing, 0, sizeof(splicing));
    splicing = strcat(splicing, "GET ");
    splicing = strcat(splicing, content);
    splicing = strcat(splicing, " HTTP/1.1\r\n");
    splicing = strcat(splicing, "Host: ");
    splicing = strcat(splicing, url);
    splicing = strcat(splicing, "\r\n\r\n");

    char recv_buf[0x1000] = { 0 };

    int ret_num = send(client, splicing, strlen(splicing), 0);
    free(splicing);

    while (1)
    {
        auto log = recv(client, recv_buf, 0x1000, 0);
        if (log==0)
        {
            break;
        }
    }

    return recv_buf;
}

//编码转换
char* utf8_to_gb(char* utf8String)
{

    if (!utf8String) return 0;

    wchar_t* unicode_buf = NULL;

    int ret_len = 0;

    ret_len = MultiByteToWideChar(CP_UTF8, 0, utf8String, -1, NULL, 0);
    unicode_buf = (wchar_t*)malloc(ret_len * sizeof(wchar_t));
    if (ret_len)
    {
        ret_len = MultiByteToWideChar(CP_UTF8, 0, utf8String, -1, unicode_buf, ret_len);

        ret_len = WideCharToMultiByte(CP_ACP, 0, unicode_buf, -1, NULL, 0, NULL, 0);

        char* gbk_buf = (char*)malloc(strlen(utf8String));
        if (gbk_buf)
        {
            memset(gbk_buf, 0, sizeof(gbk_buf));
            ret_len = WideCharToMultiByte(CP_ACP, 0, unicode_buf, -1, gbk_buf, ret_len, NULL, 0);
            free(unicode_buf);
            free(utf8String);
            return gbk_buf;
        }
    }
    return 0;
}

//网页源码内容查询
void find_gbk(char* gbk)
{
    char* find_buf = new char[0x10]{ 0 };
    printf("要查询的内容:");
    scanf_s("%s", find_buf, 0x10);
    char* ret_buf=strstr(gbk, find_buf);
    delete(find_buf);
    strtok(ret_buf, "\"");
    printf("%s\n", ret_buf);
    return ;
}

void http(const char* Url)
{
    //网址截取
    char* url = (char*)malloc(strlen(Url));
    strcpy(url, Url);
    url = &url[8];
    char* url2 = (char*)malloc(strlen(url));
    strcpy(url2, url);
    char* my_url=strtok(url, (char*)"/");
    char* content = strstr(url2, (char*)"/");

    url = NULL;
    free(url);
    url2 = NULL;
    free(url2);

    char* ip = get_ip(my_url);
    if (!ip)
    {
        printf("ip获取失败");
        return;
    }

    char* ret_buf = access_url(my_url, ip, content);
    if (!ret_buf)
    {
        printf("网站访问失败");
        return;
    }

    char* html_txt = strstr(ret_buf, "\r\n\r\n");
    char* html_gbk = utf8_to_gb(html_txt);
    if (!html_gbk)
    {
        printf("编码转换失败");
        return;
    }
    find_gbk(html_gbk);

    return;
}

char* _http(char* url)
{

    //解析请求头
    //域名解析
    //发送请求头
    //接收请求数据
    //可选解码

    if (!url) return 0;

    
    char* pos=strstr(&url[7], "/");
    if (!pos) return 0;
    pos[0]=0;
    char* domain = &url[7];
    
    
    
    in_addr *addr=0;
    hostent* host = gethostbyname(domain);
    ULONG ipaddr = 0;
    if (host)
    {
        addr = (in_addr*)host->h_addr_list[0];
        ipaddr = addr->S_un.S_addr;
        //printf("ip=%d\n", addr->S_un.S_addr);
    }
    
    if (!ipaddr) return 0;

    SOCKET client = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
    check_result(client == INVALID_SOCKET, "网络环境初始化失败2");

    //绑定套接字
    sockaddr_in client_address{ AF_INET, htons(80) };
    client_address.sin_addr.S_un.S_addr = ipaddr;

    int result = connect(client, (sockaddr*)&client_address, sizeof(sockaddr_in));
    check_result(client == SOCKET_ERROR, "绑定套接字失败");
    char qh[1024] = {0};
    sprintf(qh, "GET /%s  HTTP/1.1\r\nHOST: %s\r\n\r\n", (char*)(pos + 1), domain);

    int ret_num = send(client, qh, strlen(qh), 0);
    if (ret_num)
    {
        char* buf = (char*)malloc(4096);
        if (buf)
        {
            memset(buf, 0, 4096);
            int rlen = 0;
            while (true)
            {
                rlen = recv(client + rlen, buf, 4096, 0);
                if (rlen <= 0) break;

            }

            return utf8_to_gb(buf);

        }

    }

    return 0;
}

int main()
{
    WSADATA Wsa{};
    WSAStartup(MAKEWORD(2, 2), &Wsa);

    char url[] = "http://widget.seniverse.com/api/weather?flavor=slim&location=WX4FBXXFKE4F&geolocation=enabled&language=zh-chs";

    char* html=_http(url);
    if (html)
    {
        printf("html=%s\n", html);
    }
    char* now=strstr(html, "now\":{\"text\":\"");
    now += 14;
    if (now)
    {
        printf("今日天气:%c\n", now);
    }
    free(html);

    system("pause"); 

    //WSACleanup();

    return 0;
}