爬取360指数三部曲
- 获取360用户的token
- 通过账号、密码、token模拟登陆,获取cookie
- 设置cookie,爬取360指数
代码实现
<?php
class CrawlerOf360Data
{
private $curl;
function __construct()
{
$this->init_curl_client();
}
function init_curl_client() {
$this->curl = curl_init();
curl_setopt($this->curl, CURLOPT_URL, "https://login.360.cn/");
curl_setopt($this->curl, CURLOPT_TIMEOUT, 60);
curl_setopt($this->curl, CURLOPT_COOKIEJAR, dirname(__FILE__).'/cookie.txt');
curl_setopt($this->curl, CURLOPT_COOKIEFILE, dirname(__FILE__).'/cookie.txt');
curl_setopt($this->curl, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($this->curl, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($this->curl, CURLOPT_RETURNTRANSFER, 1);
}
function get360Data($name) {
//step1:get token
$_tokenPostFields = array(
'callback' => 'jQuery18309010124561026427_1524021670433',
'src'=> 'pcw_360index',
'from'=> 'pcw_360index',
'charset'=> 'UTF-8',
'requestScema'=> 'https',
'o'=> 'sso',
'm'=> 'getToken',
'userName'=> 'username',
'_'=> time() * 1000
);
curl_setopt($this->curl, CURLOPT_POST, true);
curl_setopt($this->curl, CURLOPT_POSTFIELDS, $_tokenPostFields);
$_tokenResponse = curl_exec($this->curl);
$tokenJson = json_decode(preg_split('/[()]/', $_tokenResponse)[1], true);
//step2:get cookies
$_postLoginFields = array(
'callback' => 'QiUserJsonp21670482',
'func' => 'QiUserJsonp21670482',
'proxy' => 'https://trends.so.com/psp_jump.html',
'type' => 'normal',
'src'=> 'pcw_360index',
'from'=> 'pcw_360index',
'charset'=> 'UTF-8',
'requestScema'=> 'https',
'o'=> 'sso',
'm'=> 'login',
'lm'=> '0',
'userName'=> 'username',
'account'=> 'username',
'password'=> 'aeeb3b46c69459542c1457ea3b3a4bc0(md5 32位小写)',
'token'=> $tokenJson['token'],
'_'=> time() * 1000
);
curl_setopt($this->curl, CURLOPT_POSTFIELDS, $_postLoginFields);
curl_exec($this->curl);
//step3:get the 360data
$cookieFileContents = file_get_contents(dirname(__FILE__).'/cookie.txt');
//拼接cookie字符串
$cookieArr = explode("\n", $cookieFileContents);
$cookie = '';
foreach ($cookieArr as $data) {
$dataArr = explode("\t", $data);
if ($dataArr[0] == '.360.cn' || $dataArr[0] == '#HttpOnly_.360.cn') {
$cookie .= "{$dataArr[5]}=$dataArr[6];";
}
}
$url = "https://trends.so.com/index/overviewJson?area=" . urldecode('全国') . '&q=' . urldecode($name);
curl_setopt($this->curl, CURLOPT_COOKIE, $cookie);
curl_setopt($this->curl, CURLOPT_URL, $url);
$result = curl_exec($this->curl);
echo $result;
}
}
补充说明
- step1:
-获取token的response,如下所示,解析字符串,获取token即可:login.setSigCallback({"errno":0,"errmsg":"","token":"638f6d593b12d10b"})
- step2:
-模拟登陆中,password为md5后的32位小写字符串。
-模拟登陆成功后,会获得如下的cookie信息:
# Netscape HTTP Cookie File
# http://curl.haxx.se/docs/http-cookies.html
# This file was generated by libcurl! Edit at your own risk.
.360.cn TRUE / FALSE 0 Q u%3D%25O9%25R3%25Q6%25QQ%25PO%25P4%25P8%25SQ%26n%3D%25O9%25R3%25Q6%25QQ%25PO%25P4%25P8%25SQ%26le%3DnzyhM3IcMT9hMlH0ZQDmBGxhozI0%26m%3D%26qid%3D251535385%26im%3w1_t01923d359dad425928%26src%3Dpcw_360index%26t%3D1
#HttpOnly_.360.cn TRUE / FALSE 0 T s%3D45ebc94e99d3b53fce0c57f504b2f070%26t%3D1524064257%26lm%3D%26lf%3D4%26sk%3Dd0ad528d2a9e9ef22ee3477128691bf1%26mt%3D1524064257%26rc%3D%26v%3D2.0%26a%3D0
- step3:
-在step3时,需要设置cookie,具体cookie文件中最后两行的拼接而成。
Q\T分别为key,紧跟着Q\T后面即对应的value.
eg:Q=u%3D%25O9%25R3%25Q6%25QQ%25PO%25P4%25P8%25SQ%26n%3D%25O9%25R3%25Q6%25QQ%25PO%25P4%25P8%25SQ%26le%3DnzyhM3IcMT9hMlH0ZQDmBGxhozI0%26m%3D%26qid%3D251535385%26im%3D1_t01923d359dad425928%26src%3Dpcw_360index%26t%3D1;T=s%3D45ebc94e99d3b53fce0c57f404b2f070%26t%3D1524064257%26lm%3D%26lf%3D4%26sk%3Dd0ad528d2a9e9ef22ee3477128691bf1%26mt%3D1524064257%26rc%3D%26v%3D2.0%26a%3D0;
-通过下面的链接测试
https://trends.so.com/index/overviewJson?area=全国&q=微信
可成功得到如下JSON:
{"status":0,"data":[{"query":"\u5fae\u4fe1","data":{"week_year_ratio":">1000%","month_year_ratio":">1000%","week_chain_ratio":"16.18%","month_chain_ratio":"19.46%","week_index":8674687,"month_index":8455856}}],"msg":"success"}