#!/usr/bin/python
import os
fh = open("code.txt", "r")
print "start crawling ....";
for line in fh:
line = line.strip()
print line
os.system("php worker_tianyi.php %d &" % int(line))
fh.close()
print "crawling started ...";
http://open.189.cn/index.php?m=api&c=index&a=show&id=418
public function fetch_api_data($region, $word) {
$page = 1;
$form = array();
$form['app_id'] = '****';
$form['access_token'] = '*****';
while(1) {
$reqxml = '
<?xml version="1.0" encoding="UTF-8"?>
<reqXML version="1.0">
<appKey>****</appKey>
<apiName>cominfo</apiName>
<apiMethod>getChaHaoInfo</apiMethod>
<timestamp>' . date('Y-m-d H:i:s') . '</timestamp>
<params>
<param name="region" value="' . $region . '" />
<param name="content" value="' . $word . '" />
<param name="Page" value="' . $page . '" />
<param name="PageSize" value="10" />
<param name="deviceid" value="18930823591" />
</params>
</reqXML>';
$form['reqXml'] = $reqxml;
$query = http_build_query($form);
$result = $this->url->get_file('http://api.189.cn/besttone/getChaHaoInfo?' . $query);
if($result) {
$result = $this->parse_xml($result);
if(!$result) {
return;
}
}
sleep(10);
$page ++;
}
}
public function parse_xml($result) {
$xml = simplexml_load_string($result);
$xml = $xml->ResultContent;
$has_more = false;
foreach($xml[0] as $ContentTags) {
$has_more = true;
$tag = $ContentTags->ContentTag[1]->propertyValue;
$number = $ContentTags->ContentTag[3]->propertyValue;
$this->add_data($number, $tag);
}
return $has_more;
}