基于Charles数据采集自动化的一次实践

前提

数据采集，一般直接调用数据传输接口。当数据传输接口的调用过于复杂时，比如：请求参数多层或自定义加密等，这个时候不妨考虑基于第三方代理软件直接拦截传输的数据。

拦截传输的数据，完整流程：设置第三方代理、程序模拟人操作、输出第三方代理软件拦截的数据、解析并存储数据。本文的第三方代理软件为Charles，故以上流程基于Charles进行演示。

设置第三方代理

进入Charles主页面后，如下依次点击，查看代理的IP与监控的端口。

如果使用Selenium等模拟人操作网页，可忽略此步；如果使用Appium等模拟人操作手机APP页面，不可忽略此步。

模拟人操作

可使用Selenium等模拟人操作网页、使用Appium等模拟人操作手机APP页面等，此处不作累述。

输出第三方代理软件拦截的数据

默认情况下，Charles会展示所有拦截的请求。

为方便查看数据接口且提高数据处理的性能，Charles提供请求过滤功能；可如下依次点击。

Charles提供分钟级的请求自动导出功能，可指定导出路径和导出文件的格式。

解析并存储数据

以“JSON Session File”为例，数据结构如下，可根据需要进行解析。

{
	"status": "COMPLETE",
	"method": "POST",
	"protocolVersion": "HTTP/1.1",
	"scheme": "http",
	"host": "210.52.217.135",
	"port": 80,
	"actualPort": 80,
	"path": "/cloudquery.php",
	"query": null,
	"tunnel": false,
	"keptAlive": false,
	"webSocket": false,
	"remoteAddress": "210.52.217.135/210.52.217.135",
	"clientAddress": "/127.0.0.1",
	"clientPort": 56166,
	"times": {
		"start": "2021-10-08T10:52:33.578+08:00",
		"requestBegin": "2021-10-08T10:52:33.586+08:00",
		"requestComplete": "2021-10-08T10:52:33.586+08:00",
		"responseBegin": "2021-10-08T10:52:33.653+08:00",
		"end": "2021-10-08T10:52:33.654+08:00"
	},
	"durations": {
		"total": 75,
		"dns": 0,
		"connect": 7,
		"ssl": null,
		"request": 0,
		"response": 1,
		"latency": 67
	},
	"speeds": {
		"overall": 31666,
		"request": 0,
		"response": 703000
	},
	"totalSize": 2375,
	"request": {
		"sizes": {
			"headers": 334,
			"body": 1338
		},
		"mimeType": "multipart/form-data",
		"charset": null,
		"contentEncoding": null,
		"header": {
			"firstLine": "POST /cloudquery.php HTTP/1.1",
			"headers": [{
				"name": "User-Agent",
				"value": "Post_Multipart"
			},
			{
				"name": "Host",
				"value": "210.52.217.135"
			},
			{
				"name": "Accept",
				"value": "*/*"
			},
			{
				"name": "Pragma",
				"value": "no-cache"
			},
			{
				"name": "X-360-Cloud-Security-Desc",
				"value": "Scan Suspicious File"
			},
			{
				"name": "x-360-ver",
				"value": "4"
			},
			{
				"name": "Content-Length",
				"value": "1338"
			},
			{
				"name": "Content-Type",
				"value": "multipart/form-data; boundary=----------------------------a692a93e79cd"
			},
			{
				"name": "Cache-Control",
				"value": "no-cache"
			},
			{
				"name": "Connection",
				"value": "keep-alive"
			}]
		},
		"body": {
			"encoding": "base64",
			"encoded": "LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tYTY5MmE5M2U3OWNkDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9Im0iDQoNCgoEBKhewAABAADyV+KtCjNgKSDLdD2n/exxS46lIe02XmWPBtkQboV5X4TJU6aG+w+FFIVzeF9becFANl9uXrdQYKkH9ET8uKHp5ra+045buG+IjRD4DdG1d0VeMeCUsQN3kNhteJuepNe59eftz7o+2beqmSPfsfd4Zw3eYqw+KH4FB8wcd4F7GmmMp2phzgaZE18GfH83byQWMhWSKMQziED9DDiie/m4clnY9gnmiFcIZC5Wm//n+Ck3o5xQjHqB1Jd6c2rPHR1gGPVk3JNBCXiuWGB5e/OpvwWpBlze3ICJQ3X0yDR18xPrKXdqOqP/CKGu3EkSj5OzY0eHdoSHLddiglYr1Htfn8WXVKfQQIS3v3K4s+ZXB5ffuoq5X67msU55yyHZgqRejf0XpcJ3iiW1VldoDUDCxog1NS95o5zhCk+IUstm59AuK9WWhmax0QaQ3dQMPSIz2PiTSYbYrIpOh+3sOBd77MMZmxD2OAziybPoWT79XmxsFrwL/dI2bBUvZ0Ve19AHlTMdUYYyFPkbYzvMoAAHbZOKRn4Tvp8C1G5e5Qosw4ccY+Mv+214cOdAdA3x/nDvOQxs9DQW1wCfKY38vmYawbLT2myNk7aKYNXUu6fLbZEapDOMJxTu3d9rcY+fE8rogbYIrDR45nky+7Lpvv8mJ//GLpCQjeW4oSF7Boywe4rwi/hfroLinaorR6EpN1E63+Y1eYgjqkVh2fe9AB4EEnhhWimx0xE6/c15SpnNNXr+3LyZqKv38E8jo9qVw7sqEAzG1lKzZRqcFeovJntqDgMoy1DTgRfomYpV4bOWVIh8636e1YwJvc5iXaZ7n/a2yhgMsVqxXyblNHWOH82bR4aDyS31F3pdk6MD2WXtTkiSjfNyaZ2stuWGlgnz2t+YVZJ4xxS3/ePjrCMY3lg29+3az/WdvS9E/5qo6poIWmgSnkE1UjDMO5UBvq0RDeVbW9XVzpKy7tVpScjTrGBPKehMwQf0+aS9Hy/NkSblmciUuIUp2aGcINRIjrrILtiYUd/POE3PBFzIxzKlc2MySKBNcMlgZmjGpTeoG/NiK4R9Z4yxkPyPvC9bCelTftEkY2KFzJtcZfXI/IujqA6X0CNeSiQlQVl1rA2EpKMLqlviQZ7m95GgqCI0YJWDaTxd8zsYbsCbAxK38psabThfbCWmcHMUjiHQbJJtw7F4+/KaRnzm+L18lmHYeY55dnZroPsOR24rRZtXTbBjlLRg2QWnkzVtlhGao/wXJVAz03LFmMTX+YiHioCou4ShnW/DaMfayhZctY93Gd5PWk3lpCk4UEPo7pJtbXXsVJu0jxzPxlk1uCfbTPz9q61dEX1QKSjdkUYtLzHM5sCx2vTEQ2FEVAVPSgOwXMJyWD1V4BLcMr3zKVnR35wg/TB7qWgNjpavEn6L5QYr/RNPHvSL7QK+Ndp29daGABr9fbEJQjTlR6NmmY8BQaET5mfB2F8+F39OZfRRACXKr7rwoeXkv7IWdMs2hN/JxJ+QpCUHhh0Zb3RmrXAVyBA4zRBosaiB6mGMQz9DLapGWmlQNXWl5kpSHZxNKWz3uRuRU/uyavpx0aHUYHFcDQotLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS1hNjkyYTkzZTc5Y2QtLQ0K"
		}
	},
	"response": {
		"status": 200,
		"sizes": {
			"headers": 236,
			"body": 467
		},
		"mimeType": "application/octet-stream",
		"charset": null,
		"contentEncoding": null,
		"header": {
			"firstLine": "HTTP/1.1 200 OK",
			"headers": [{
				"name": "Server",
				"value": "nginx"
			},
			{
				"name": "Date",
				"value": "Fri, 08 Oct 2021 02:52:15 GMT"
			},
			{
				"name": "Content-Type",
				"value": "application/octet-stream"
			},
			{
				"name": "Transfer-Encoding",
				"value": "chunked"
			},
			{
				"name": "Cache-Control",
				"value": "no-cache"
			},
			{
				"name": "pragma",
				"value": "no-cache"
			},
			{
				"name": "Expires",
				"value": "0"
			},
			{
				"name": "Proxy-Connection",
				"value": "close"
			}]
		},
		"body": {
			"encoding": "base64",
			"encoded": "CgQBv17AAAEAABECUEFJrq6Td0kAiSrClNUoidc4a+3QyOM0gJi/48D8GBcYMa4/lcmPwdvJcPivJ/RwB8yroOFPbPDvng/24w9uNMgyzdFPhkPbmC02tznpo4YqBpsHnzz/UqcLoQahAPFVpw73D6sGp1OlVasA8wb2BKsOpQ/3BasDogW5UqEOplOrUKYGpQSnBKMHp1WjAqRQ9gSkB/AH9geqAqdV9wWiVacApT+jBrwFoD+iP6I/oj+gA5sDmz+bZtp45Gz8ZOhV/3u5VaB/pHvGd6N71mejePhV6lDaTuhv/wKkA95P0APeX/MD8ADnA94d+mbRD+hUoGyiVaF8+Gb1C68/mwabBps8qlOkA/cGoQWlD6tT8A/0UqYA8FWnVaMOplOlUqNUqgO5U6NV8QOhB6FU9wf0AfZQplWmBaQPpVCqUKUGowWiA6cOp1D3AvcBoz+jBrwGmwabBpsGmweiBKs/pT+bP8Jw1GHGYqdhxHDAT/Zu3AbDYMh59mGiAN9y5UDHYMh4wlzqWfNu0EzCXadzx3DYZsRzq2LdXNQO9HOnc8MG1GPdWNxBygTURfBy5UDzcf5B8UymC5s/oj+iP5g7mA=="
		}
	}
}

补充

拦截的请求里除了数据外，还有Url、请求头、请求体等。有些网站Cookie、Session等爬虫资源更新的频率较低时，该方案可用作相应的爬虫资源的更新。