c 抓取php网页内容,C/C++: libcurl获取网页内容

安装：

$ sudo apt-get install libcurl3

$ sudo apt-get install libcurl4-openssl-dev

示例

#include

#include

#include

#include

struct url_data {

size_t size;

char* data;

};

size_t write_data(void *ptr, size_t size, size_t nmemb, struct url_data *data) {

size_t index = data->size;

size_t n = (size * nmemb);

char* tmp;

data->size += (size * nmemb);

#ifdef DEBUG

fprintf(stderr, "data at %p size=%ld nmemb=%ld\n", ptr, size, nmemb);

#endif

tmp = (char*)realloc(data->data, data->size + 1); /* +1 for '\0' */

if(tmp) {

data->data = tmp;

} else {

if(data->data) {

free(data->data);

}

fprintf(stderr, "Failed to allocate memory.\n");

return 0;

}

memcpy((data->data + index), ptr, n);

data->data[data->size] = '\0';

return size * nmemb;

}

char *handle_url(char* url) {

CURL *curl;

struct url_data data;

data.size = 0;

data.data = (char*)malloc(4096); /* reasonable size initial buffer */

if(NULL == data.data) {

fprintf(stderr, "Failed to allocate memory.\n");

return NULL;

}

data.data[0] = '\0';

CURLcode res;

curl = curl_easy_init();

if (curl) {

curl_easy_setopt(curl, CURLOPT_URL, url);

curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);

curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);

res = curl_easy_perform(curl);

if(res != CURLE_OK) {

fprintf(stderr, "curl_easy_perform() failed: %s\n",

curl_easy_strerror(res));

}

curl_easy_cleanup(curl);

}

return data.data;

}

int main(int argc, char* argv[]) {

char* data;

if(argc < 2) {

fprintf(stderr, "Must provide URL to fetch.\n");

return 1;

}

data = handle_url(argv[1]);

if(data) {

printf("%s\n", data);

free(data);

}

curl_global_cleanup();

return 0;

}

编译、运行：

$ gcc -g test.c -lcurl

$ ./a.out www.baidu.com

示例 (多线程)

#include

#include

#include

#include

#include

struct url_data {

size_t size;

char* data;

};

size_t write_data(void *ptr, size_t size, size_t nmemb, struct url_data *data) {

size_t index = data->size;

size_t n = (size * nmemb);

char* tmp;

data->size += (size * nmemb);

#ifdef DEBUG

fprintf(stderr, "data at %p size=%ld nmemb=%ld\n", ptr, size, nmemb);

#endif

tmp = (char*)realloc(data->data, data->size + 1); /* +1 for '\0' */

if(tmp) {

data->data = tmp;

} else {

if(data->data) {

free(data->data);

}

fprintf(stderr, "Failed to allocate memory.\n");

return 0;

}

memcpy((data->data + index), ptr, n);

data->data[data->size] = '\0';

return size * nmemb;

}

char *handle_url(char* url) {

CURL *curl;

struct url_data data;

data.size = 0;

data.data = (char*)malloc(4096); /* reasonable size initial buffer */

if(NULL == data.data) {

fprintf(stderr, "Failed to allocate memory.\n");

return NULL;

}

data.data[0] = '\0';

CURLcode res;

curl = curl_easy_init();

if (curl) {

curl_easy_setopt(curl, CURLOPT_URL, url);

curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);

curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);

res = curl_easy_perform(curl);

if(res != CURLE_OK) {

fprintf(stderr, "curl_easy_perform() failed: %s\n",

curl_easy_strerror(res));

}

curl_easy_cleanup(curl);

}

return data.data;

}

void threadFunc() {

char *url = "www.baidu.com";

char* data;

data = handle_url(url);

if(data) {

printf("%s\n", data);

free(data);

}

}

int main(int argc, char* argv[]) {

for(int i=0; i<10000; ++i)

{

std::thread *t = new std::thread(threadFunc);

t->join();

delete t;

}

curl_global_cleanup();

return 0;

}

g++ -std=c++11 test.cpp -lcurl -pthread

C++版本