最近要写一个代理程序,软件最终要跑在嵌入式设备上,其中一部分是需要做一个简单爬虫程序,用来和嵌入式设备上的Web做信息交互。我不想用第三方的任何库,如是简单看了下http协议,用一天时间实现了http协议的客户端,实现Get、Post、UpFile(文件上传)等常用操作,需要完善的部分是Cookie没有自动提取和传输,需要自己手动处理,朋友们可以完善吧!写个日志,便于日后参考!希望对朋友有参考。
由于最终要在嵌入式设备上运行,所以用 #define Windows 来区分,Socket部分Window和Linux还是有差别的,程序兼容了Windows和Liunx。
通信类头文件
#pragma once
#include<string>
#ifndef WINDOWS
#define WINDOWS
#endif
#ifdef WINDOWS
#include<WINSOCK2.H>
#include <WS2tcpip.h>
#else
#include <iconv.h>
#include <netinet/in.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <string.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#endif // WINDOWS
using namespace std;
class TCPClient
{
public:
TCPClient(string host);
~TCPClient();
bool ConnectToServer();
bool SendData(const char * sendBuffer,int dataLen);
string RecvData();
private:
#ifdef WINDOWS
SOCKET clientSocket;
#else
int clientSocket;
#endif
string hostAddr;
bool isConnected;
};
封装基础通信类cpp
#include "TCPClient.h"
#include<vector>
#include<errno.h>
#ifdef WINDOWS
#pragma warning(disable:4996)
#pragma comment(lib, "WS2_32.lib")
#endif // WINDOWS
#ifdef WINDOWS
TCPClient::TCPClient(string host)
{
hostAddr = host;
isConnected = false;
#ifdef WINDOWS
WSAData wsdata;
if (WSAStartup(MAKEWORD(2, 2), &wsdata) != 0)
{
WSACleanup();
printf("WSAStartup failured \r\n");
}
else
{
printf("WSAStartup OK\r\n");
}
#endif
}
bool TCPClient::ConnectToServer()
{
if (isConnected)
return true;
clientSocket = socket(AF_INET, SOCK_STREAM, 0);
if (clientSocket == -1)
{
printf("sockclient create fail ! \n");
return false;
}
struct sockaddr_in dest_addr; /* 目的地址*/
dest_addr.sin_family = AF_INET; /* host byte order */
dest_addr.sin_port = htons(80); /* short, network byte order */
#ifdef WINDOWS
dest_addr.sin_addr.s_addr = inet_addr(this->hostAddr.c_str());
#else
inet_pton(AF_INET, this->hostAddr.c_str(), &dest_addr.sin_addr.s_addr);
#endif // WINDOWS
if (connect(clientSocket, (struct sockaddr *)&dest_addr, sizeof(struct sockaddr)) < 0)
{
#ifdef WINDOWS
closesocket(clientSocket);
printf("connect controlServer failured%d\r\n", GetLastError());
#else
close(clientSocket);
printf("connect controlServer failured\r\n");
#endif
return false;
}
isConnected = true;
return true;
}
TCPClient::~TCPClient()
{
#ifdef WINDOWS
closesocket(clientSocket);
WSACleanup();
#else
close(clientSocket);
#endif
}
bool TCPClient::SendData(const char *sendBuffer, int dataLen)
{
if (!ConnectToServer())
return false;
int sendLen = 0;
int oneLen = 0;
while (sendLen < dataLen)
{
oneLen = send(this->clientSocket, sendBuffer + sendLen, dataLen - sendLen, 0);
if (oneLen <= 0)
{
if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)
{
//这几种错误码,认为连接是正常的,继续接收
}
else {
//printf("%d", WSAGetLastError());
return 0;
}
break;
}
else
sendLen += oneLen;
}
return true;
}
string TCPClient::RecvData()
{
char recvData[1000] = { 0 };
string responseData = "";
int recLen = 0;
int recLenOfAll = 0;
while ((recLen = recv(this->clientSocket, recvData, sizeof(recvData) - 1, 0)) > 0)
{
responseData.append(recvData);
memset(recvData, 0, 1000);
//ZeroMemory(recvData, 1000);
recLenOfAll += recLen;
}
string resp;
#ifdef WINDOWS
resp =responseData.c_str();
closesocket(this->clientSocket);
#else
resp = (responseData.c_str());
close(this->clientSocket);
#endif
isConnected = false;
return resp;
}
WebRequest头文件
#pragma once
#include<iostream>
#include<thread>
#include<mutex>
#include<map>
#include<string>
#include "TCPClient.h"
using namespace std;
enum RequestType
{
Post, Get
};
class WebRequest
{
private:
string SendData(string method, string path, map<string, string> paramters, string referUrl);
string SendData(string method, string path, string bodyInfo, string referUrl);
string SendData(string method, string path, string referUrl);
public:
WebRequest(string host);
~WebRequest();
string GetData(string path, map<string, string> paramters, string referUrl);
string GetData(string path, string referUrl);
string PostData(string path, map<string, string> paramters, string referUrl);
string PostData(string path, string referUrl);
string GetData(string path, string bodyInfo, string referUrl);
string PostData(string path, string bodyInfo, string referUrl);
string UpFile(string path, string FileName);
public:
map<string, string> Headers;
RequestType requestType;
private:
string GetHeadContent();
TCPClient *tcpClient;
string hostAddr;
};
webrequest.cpp
#include "WebRequest.h"
#include<vector>
#include<fstream>
WebRequest::WebRequest(string host)
{
this->hostAddr = host;
//Headers.insert(pair<string,string>("A", "B"));
map<string, string> *HeadersTest = new map<string, string>();
Headers["Accept"]="*/*";
Headers["Host"] = host;
Headers["Accept-Language"] = "zh-CN";
Headers["Accept-Encoding"] = "gzip, deflate";
Headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko";
Headers["Connection"] = "Keep-Alive";
tcpClient = new TCPClient(host);
}
WebRequest::~WebRequest()
{
delete tcpClient;
}
string WebRequest::GetHeadContent()
{
string headContent = "";
for (const auto& kv : this->Headers)
{
if (kv.second.size() == 0)
continue;
headContent.append(kv.first);
headContent.append(": ");
headContent.append(kv.second);
headContent.append("\r\n");
}
return headContent;
}
string WebRequest::SendData( string method, string path, string bodyInfo, string referUrl)
{
string headStr = "";
headStr.append(method+" " + path + " HTTP/1.1\r\n");
headStr.append(GetHeadContent());
if (referUrl.length() > 0)
headStr.append("Referer: http://"+hostAddr + referUrl +"\r\n");
headStr.append("Content-Length: " + to_string(bodyInfo.size()));
headStr.append("\r\n\r\n");
headStr.append(bodyInfo);
if (tcpClient->SendData(headStr.c_str(), headStr.length()))
return tcpClient->RecvData();
else
return "发送请求失败!";
}
string WebRequest::GetData(string path, map<string, string> paramters, string referUrl)
{
string recstr = this->SendData("GET", path, paramters, referUrl);
//this->temRecStr = recstr;
return recstr;
}
string WebRequest::PostData(string path, map<string, string> paramters, string referUrl)
{
return this->SendData("POST", path, paramters, referUrl);
}
string WebRequest::GetData(string path, string referUrl)
{
return this->SendData("GET", path, referUrl);
}
string WebRequest::PostData(string path, string referUrl)
{
return this->SendData("POST", path, referUrl);
}
string WebRequest::GetData(string path, string bodyInfo, string referUrl)
{
return this->SendData("GET", path, bodyInfo, referUrl);
}
string WebRequest::PostData(string path, string bodyInfo, string referUrl)
{
return this->SendData("POST", path, bodyInfo, referUrl);
}
string WebRequest::SendData(string method, string path, map<string,string> paramters,string referUrl)
{
string headStr = "";
headStr.append(method+" "+ path+" HTTP/1.1\r\n");
headStr.append(GetHeadContent());
if (referUrl.length() > 0)
headStr.append("Referer: http://" +hostAddr+ referUrl + "\r\n");
string bodyInfo = "";
for (const auto& kv : paramters)
{
//if(TString::endsWith(path,"&"))
if (bodyInfo.length() > 0)
bodyInfo.append("&");
bodyInfo.append(kv.first);
bodyInfo.append("=");
bodyInfo.append(kv.second);
}
headStr.append("Content-Length: " + to_string(bodyInfo.size()));
headStr.append("\r\n\r\n");
headStr.append(bodyInfo);
if (this->tcpClient->SendData(headStr.c_str(), headStr.length()))
return tcpClient->RecvData();
else
return"请求失败!";
}
string WebRequest::SendData(string method, string path, string referUrl)
{
string headStr = "";
headStr.append(method + " " + path + " HTTP/1.1\r\n");
headStr.append(GetHeadContent());
if (referUrl.length() > 0)
headStr.append("Referer: http://" + hostAddr + referUrl + "\r\n");
headStr.append("Content-Length:0" );
headStr.append("\r\n\r\n");
if (this->tcpClient->SendData(headStr.c_str(), headStr.length()))
return tcpClient->RecvData();
else
return"请求失败!";
}
string WebRequest::UpFile(string path, string fileName)
{
string boundary = "---------------------------7d33a816d302b6\r\n";
this->Headers["Content-Type"] = "multipart/form-data;boundary=---------------------------7d33a816d302b6";// +boundary;
string bodyInfo = "";
fstream fs;
fs.open(fileName, ios::binary | ios::in);
if (!fs)
{
cout << "文件不存在!" << fileName << endl;
return "上传文件不存在!";
}
fs.seekg(0, fs.end);
int fsLen=fs.tellg();
bodyInfo.append(boundary);//文件内容开始,
bodyInfo.append("Content-Disposition: form-data; name=\"fx_19V_11.10.128.14.uot\"; filename=\"" + fileName + "\"\r\n");
bodyInfo.append("Content-Type: application/octet-stream\r\n");
string sendContent = "";
sendContent.append("POST " + path + " HTTP/1.1\r\n");
sendContent.append(GetHeadContent());
sendContent.append("Content-Length: " + to_string(fsLen + bodyInfo.length() + boundary.length()));
sendContent.append("\r\n\r\n");
sendContent.append(bodyInfo);
const char* sendBuffer = sendContent.c_str();
if (!tcpClient->SendData(sendBuffer, sendContent.length()))
return "发送请求失败";
//这里读取文件,发送文件
fs.seekg(0, fs.beg);
char fsBuffer[2000];
int readLen = 0;
int sendAll = 0;
while (!fs.eof())
{
fs.read(fsBuffer, 2000);
readLen=fs.gcount();
if (!tcpClient->SendData(fsBuffer, readLen))
break;
else
{
sendAll += readLen;
cout <<"已发送:"<< sendAll << endl;
}
}
fs.close();
if(!tcpClient->SendData(("\r\n"+boundary).c_str(), boundary.length()))//协议结束
return "发送请求失败";
return tcpClient->RecvData();
}