#OBJS=handledata.o source.o
OBJS=source.o
main:$(OBJS)
g++ -g -o lq-client $(OBJS) -l curl -l pthread -std=c++11
#handledata.o:handledata.c
# g++ -c -Wall handledata.c -I /usr/local/include/curl -std=c++11
source.o:source.cpp
g++ -g -c -Wall source.cpp -l curl -l pthread -std=c++11
clean:
rm -f lq-client $(OBJS)
http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=3000&sort=symbol&asc=1&node=sz_a&symbol=&_s_r_a=page
#include <stdio.h>
#include <iconv.h>
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
#include <vector>
#include <map>
#include <string>
#include <sstream>
#include <iostream>
#include <regex>
#include <sys/time.h>
#include <curl/curl.h>
using namespace std;
struct MemoryStruct
{
char *memory;
size_t size;
};
static size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
size_t realsize = size * nmemb;
struct MemoryStruct *mem = (struct MemoryStruct *)userp;
mem->memory = (char *) realloc(mem->memory, mem->size + realsize + 1);
if (mem->memory == NULL)
{
printf("not enough memory (realloc returned NULL)\n");
return 0;
}
memcpy(&(mem->memory[mem->size]), contents, realsize);
mem->size += realsize;
mem->memory[mem->size] = 0;
return realsize;
}
int code_convert(char *from_charset, char *to_charset, char *inbuf, size_t inlen,
char *outbuf, size_t outlen) {
iconv_t cd;
char **pin = &inbuf;
char **pout = &outbuf;
cd = iconv_open(to_charset, from_charset);
if (cd == 0)
return -1;
memset(outbuf, 0, outlen);
if (iconv(cd, pin, &inlen, pout, &outlen) == -1)
return -1;
iconv_close(cd);
return 0;
}
int u2g(char *inbuf, size_t inlen, char *outbuf, size_t outlen) {
return code_convert("utf-8", "gb2312", inbuf, inlen, outbuf, outlen);
}
int g2u(char *inbuf, size_t inlen, char *outbuf, size_t outlen) {
return code_convert("gb2312", "utf-8", inbuf, inlen, outbuf, outlen);
}
void FreeStrBuf(char **ppBuf)
{
if (NULL != (*ppBuf))
{
free((*ppBuf));
(*ppBuf) = NULL;
}
}
void FreeStrBuf(int **ppBuf)
{
if (NULL != (*ppBuf))
{
free((*ppBuf));
(*ppBuf) = NULL;
}
}
pthread_mutex_t sum_mutex;
class shares_data
{
public:
shares_data()
{
name = "";
}
std::string name;
};
inline void NEXT(const string&T, vector<int>&next) {
next[0] = -1;
for (int i = 1; i<T.size(); i++) {
int j = next[i - 1];
while (T[i - 1] != T[j] && j >= 0) j = next[j];
if (T[i - 1] == T[j]) next[i] = j + 1;
else next[i] = 0;
}
}
inline string::size_type KMP(const string&S, const string&T) {
vector<int>next(T.size());
NEXT(T, next);
string::size_type index, count = 0;
for (index = 0; index<S.size(); ++index)
{
int pos = 0;
string::size_type iter = index;
while (pos<T.size() && iter<S.size())
{
if (S[iter] == T[pos]) { ++iter; ++pos; }
else {
if (pos == 0) ++iter;
else pos = next[pos - 1] + 1;
}
}
if (pos == T.size() && (iter - index) == T.size()) ++count;
if (count == 1)
{
break;
}
}
return count;
}
std::map<std::string, std::string> shares_map;
void* getdata(void * num)
{
int num_int = 0;
num_int = *((int*) &num);
char *prelink = "http://q.10jqka.com.cn/index/index/board/all/field/zdf/order/desc/page/";
char *suflink = "/ajax/1/";
char buf[10] = { NULL };
sprintf(buf, "%d", num);
char *charnum = buf;
char *templink = (char*) malloc(strlen(charnum) + strlen(suflink) + 1);
strcpy(templink, charnum);
strcat(templink, suflink);
char *totallink = (char*) malloc(strlen(prelink) + strlen(templink) + 1);
strcpy(totallink, prelink);
strcat(totallink, templink);
CURL *curl;
CURLcode res;
struct MemoryStruct chunk;
chunk.memory = (char*) malloc(1);
chunk.size = 0;
curl = curl_easy_init();
if (curl)
{
curl_easy_setopt(curl, CURLOPT_URL, totallink);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10);
curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "gzip");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &chunk);
res = curl_easy_perform(curl);
if (res != CURLE_OK)
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
curl_easy_cleanup(curl);
}
char * databuf = 0;
databuf = (char*) malloc((int) 2 * (chunk.size));
g2u(chunk.memory, chunk.size, databuf, 2 * (chunk.size));
std::string line;
std::stringstream stream;
stream << databuf;
enum STATEFLAG
{
SHARES_ID,
SHARES_NAME,
};
STATEFLAG linecurflag = SHARES_ID;
std::pair<std::string, std::string> shares_pair;
std::string shares_code = "";
std::string shares_name = "";
while (std::getline(stream, line))
{
switch (linecurflag)
{
case SHARES_ID:
{
if (KMP(line, " target=\"_blank\">"))
{
linecurflag = SHARES_NAME;
shares_code = line.substr(89,6);
}
break;
}
case SHARES_NAME:
{
shares_pair.first = shares_code;
shares_pair.second = shares_code;
shares_map.insert(shares_pair);
shares_code = "";
shares_name = "";
linecurflag = SHARES_ID;
break;
}
default:
{
continue;
}
}
}
free(databuf);
free(chunk.memory);
FreeStrBuf(&totallink);
FreeStrBuf(&templink);
pthread_exit(0);
}
int main(void)
{
curl_global_init(CURL_GLOBAL_DEFAULT);
unsigned long start_time, end_time;
time_t t;
start_time = time(&t);
pthread_t thread_id;
std::vector<pthread_t> thread_id_list;
thread_id_list.clear();
pthread_mutex_init(&sum_mutex,NULL);
shares_map.clear();
int num = 1;
int endpages = 147;
for (; num <= endpages; ++num)
{
pthread_create(&thread_id, NULL, getdata, (void*) num);
thread_id_list.push_back(thread_id);
}
for (int i = 0; i < thread_id_list.size(); ++i)
{
pthread_join(thread_id_list[i], NULL);
}
curl_global_cleanup();
pthread_mutex_destroy(&sum_mutex);
for (std::map<std::string, std::string>::iterator it = shares_map.begin(); it != shares_map.end();++it)
{
}
printf("The num of pages is:%d\n", shares_map.size());
end_time = time(&t);
printf("The start time is:%d\n", start_time);
printf("The end time is:%d\n", end_time);
printf( "The total time used is:%d\n" ,(end_time - start_time));
return 0;
}