2018/05/29 修改抓取编码gb2312改gb18030
项目需要行政区域三级联动，刚好写个爬虫练练手。
Laravel 框架，安装的两个库
composer require guzzlehttp/guzzle
composer require symfony/dom-crawler创建表
DROP TABLE IF EXISTS `area`;
CREATE TABLE `area` (
`id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
`name` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`parent_id` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
DROP TABLE IF EXISTS `crawler`;
CREATE TABLE `crawler` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`status` int(11) DEFAULT '0',
`data` text COLLATE utf8mb4_unicode_ci,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
创建对应Model
App\Model\Area.php
<?php
namespace App\Model;
use Illuminate\Database\Eloquent\Model;
class Area extends Model
{
public $timestamps = false;
protected $table = 'area';
protected $keyType = 'string';
protected $fillable = [
'id', 'name', 'parent_id',
];
}App\Model\Crawler.php
<?php
namespace App\Model;
use Illuminate\Database\Eloquent\Model;
class Crawler extends Model
{
public $timestamps = false;
protected $table = 'crawler';
protected $fillable = [
'id', 'status', 'data'
];
}app/Console/Kernel.php 添加
protected $commands = [
'App\Console\Commands\CityCrawler',
];新建 App\Console\Commands\CityCrawler.php
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Symfony\Component\DomCrawler\Crawler;
use App\Model\Area;
use App\Model\Crawler as CrawlerTask;
use GuzzleHttp\Psr7;
use GuzzleHttp\Exception\RequestException;
// 流程：
// 1. func top 抓取行政区域省级, 每个省链接生成一次抓取任务，保存到任务表crawler。
// 2. 循环抓取
// 1). 读取一条任务 select * from crawler where status = 0 limit 1; update crawler set status = 1 where id = 本次任务id;
// 2). 根据任务类型调用抓取方法 如 镇抓取：crawler_towntr 区抓取crawler_districts 城市抓取crawler_citys
// 抓取方法中保存抓取到的行政区域数据到area表，并根据抓取行政区域下一级生成下一次抓取任务，存放任务表
class CityCrawler extends Command
{
protected $signature = 'street:crawler';
protected $description = 'Street Crawler';
protected $start_url = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/';
protected $special_city = ['东莞市','中山市','嘉峪关市','三沙市','儋州市']; // 中国5个不设市辖区的地级市
public function handle()
{
//抓取省级行政区域
$this->top();
while (true) {
$task_model = $this->task();
if(empty($task_model)){
return $this->info("End");
}
$task= json_decode($task_model->data, true);
// 打印日志
$this->info(implode(',', array_map(function($item){
return $item['id'] . ' ' . $item['name'];
}, $task['data'])));
$status = call_user_func(array($this, 'crawler_' . $task['crawler']), $task);
if($status){
$this->finish($task_model);
}else{
var_dump($task, 'error');
return false;
}
$this->info("sleep 1");
sleep(1);
}
}
public function finish($task){
$task->status = 2;
$task->save();
}
public function task($status = 0)
{
$task = CrawlerTask::where("status", $status)->first();
$task->status = 1; // 进行中
$task->save();
return $task;
}
public function push($data)
{
$task = new CrawlerTask;
$task->data = json_encode($data);
$task->save();
}
// 第一个页面
public function top()
{
$url = $this->start_url;
$html = $this->send_http($url);
$crawler = new Crawler();
$crawler->addHtmlContent($html, 'gb18030');
$crawler->filter('.provincetr')->filter('td > a')->each(function(Crawler $node, $i) use($url) {
$text = $node->text();
$href = $node->attr('href');
$id = str_replace('.html', '', $href);
$task = [
'crawler' => 'citys',
'remark' => '省',
'url' => substr($url, 0, strrpos($url, '/')) . '/' . $href,
'data' => [ [ 'name' => $text, 'id' => $id] ],
'parent_id' => $id,
];
$this->push($task);
Area::create(
[
'id' => $id,
'name' => $text,
'parent_id' => 0
]
);
$this->info($node->attr('href'));
$this->info($text);
});
}
public function crawler_towntr($task)
{
$url = $task['url'];
if(!strpos($url, '.html')){
$this->info('为空的直辖市');
return true;
}
$html = $this->send_http($url);
$crawler = new Crawler();
$crawler->addHtmlContent($html, 'gb18030');
$crawler->filter('.towntr')->each(function(Crawler $node, $i) use ($task, $url) {
$code_node = $node->filter('td')->eq(0)->filter('a');
$name_node = $node->filter('td')->eq(1)->filter('a');
Area::create(
[
'id' => $code_node->text(),
'name' => $name_node->text(),
'parent_id' => $task['parent_id']
]
);
$this->info($code_node->text() . ' ' . $name_node->text());
});
return true;
}
public function crawler_districts($task)
{
$url = $task['url'];
$html = $this->send_http($url);
$crawler = new Crawler();
$crawler->addHtmlContent($html, 'gb18030');
$crawler->filter('.countytr')->each(function(Crawler $node, $i) use ($task, $url) {
$code_node = $node->filter('td')->eq(0)->filter('a');
$name_node = $node->filter('td')->eq(1)->filter('a');
//没有子节点
if($code_node->count() == 0){
$code_node = $node->filter('td')->eq(0);
$name_node = $node->filter('td')->eq(1);
}else{
$href = $code_node->attr("href");
$data = $task['data'];
$data[] = ['name' => $name_node->text(), 'id' => $code_node->text()] ;
$new_task = [
'crawler' => 'towntr',
'remark' => '县 区',
'url' => substr($url, 0, strrpos($url, '/')) . '/' . $href,
'data' => $data,
'parent_id' => $code_node->text(),
];
$this->push($new_task);
}
$this->info($code_node->text() . ' ' . $name_node->text());
Area::create(
[
'id' => $code_node->text(),
'name' => $name_node->text(),
'parent_id' => $task['parent_id']
]
);
});
return true;
}
public function crawler_citys($task)
{
$url = $task['url'];
$html = $this->send_http($url);
$crawler = new Crawler();
$crawler->addHtmlContent($html, 'gb18030');
$crawler->filter('.citytr')->each(function(Crawler $node, $i) use ($task, $url) {
$code_node = $node->filter('td')->eq(0)->filter('a');
$name_node = $node->filter('td')->eq(1)->filter('a');
$href = $code_node->attr("href");
$this->info($code_node->text() . ' ' . $name_node->text());
Area::create(
[
'id' => $code_node->text(),
'name' => $name_node->text(),
'parent_id' => $task['parent_id']
]
);
$data = $task['data'];
$data[] = ['name' => $name_node->text(), 'id' => $code_node->text()] ;
if(in_array($name_node->text(), $this->special_city)){
$new_task = [
'crawler' => 'towntr',
'remark' => '特别的5个省地级市',
'url' => substr($url, 0, strrpos($url, '/')) . '/' . $href,
'data' => $data,
'parent_id' => $code_node->text(),
];
}else{
$new_task = [
'crawler' => 'districts',
'remark' => '城市',
'url' => substr($url, 0, strrpos($url, '/')) . '/' . $href,
'data' => $data,
'parent_id' => $code_node->text(),
];
}
$this->push($new_task);
});
return true;
}
public function info($string, $verbosity = null)
{
$string = iconv( 'UTF-8', 'GB18030', $string); // cmd 中文gbk编码
parent::line($string, 'info', $verbosity);
}
private function send_http($url)
{
$user_agent_list = [
'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.04',
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.9 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 OPR/48.0.2685.52',
'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:56.0) Gecko/20100101 Firefox/56.0',
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
];
$user_agent = $user_agent_list[(time() % 6)];
$timeout = 5; // 秒
$client = new \GuzzleHttp\Client(['headers' => ['User-Agent' => $user_agent], 'timeout' => $timeout]);
try {
$res = $client->request('GET', $url);
$html = (string)$res->getBody();
} catch (RequestException $e) {
// 抓取中会有404状态返回，再重新请求一次。
$this->info(Psr7\str($e->getRequest()));
if ($e->hasResponse()) {
$this->info(Psr7\str($e->getResponse()));
}
$this->info("send_http timeout retry");
$this->info("sleep 2s");
sleep(2);
$res = $client->request('GET', $url);
$html = (string)$res->getBody();
}
return $html;
}
}
进目录运行
php artisan street:crawler
最后
------------------------------------------------------
数据有了全部写到一个json文件里，太大了1M多 :(
还是写成ajax从服务端读取三级联动数据。
area.js
// 1. 省加载 其他请选择
// 2. 省 change 触发加载市
// 3. 市触发加载区
// 4. 区加载触发街道
function area_init(param) {
var area = this;
area.area_not_filter = false;;
area.prompt_html = '<option value="">－请选择－</option>';
if (param.area_not_filter) {
area.area_not_filter = param.area_not_filter;
}
if (param.province) {
area.province_el = $(param.province);
}
if (param.city) {
area.city_el = $(param.city);
}
if (param.district) {
area.district_el = $(param.district);
}
if (param.street) {
area.street_el = $(param.street);
}
area.load = function() {
area.province_el.html(area.prompt_html);
area.city_el.html(area.prompt_html);
area.district_el.html(area.prompt_html);
if (area.street_el) {
area.street_el.html(area.prompt_html);
}
province_id = area.province_el.attr("data-value");
city_id = area.city_el.attr("data-value");
district_id = area.district_el.attr("data-value");
if (area.street_el) {
street_id = area.street_el.attr("data-value");
}
area.area_fill(0, 'province', area.province_el, province_id);
province_id && area.area_fill(province_id, 'city', area.city_el, city_id);
city_id && area.area_fill(city_id, 'district', area.district_el, district_id);
if (area.street_el) {
district_id && area.area_fill(district_id, 'street', area.street_el, street_id);
}
}
area.bind = function() {
area.province_el.change(function() {
area.area_fill($(this).val(), 'city', area.city_el);
area.city_el.html(area.prompt_html);
area.district_el.html(area.prompt_html);
if (area.street_el) {
area.street_el.html(area.prompt_html);
}
});
area.city_el.change(function() {
area.area_fill($(this).val(), 'district', area.district_el);
area.district_el.html(area.prompt_html);
if (area.street_el) {
area.street_el.html(area.prompt_html);
}
});
if (area.street_el) {
area.district_el.change(function() {
area.area_fill($(this).val(), 'street', area.street_el);
area.street_el.html(area.prompt_html);
});
}
};
area.area_fill = function(parent_id, level, el, active_id) {
// value='' 不请求 ajax
if (parent_id === '') {
return false;
}
area.get_area(parent_id, level, function(list) {
var province = area.prompt_html;
$.each(list, function(i, n) {
province += '<option ' + (active_id == n.id ? ' selected ' : ' ') + ' value="' + n.id + '">' + n.name + '</option>';
});
$(el).html(province);
});
}
area.get_area = function(parent_id, level, callback) {
var $url = '/ajax_area/' + level + '/' + parent_id;
$.ajax({
url: $url,
type: 'get',
success: function(res) {
callback(res);
}
});
}
area.load();
area.bind();
}
// 使用方法
// data-value 默认值
// 32 江苏省
// 320100000000 南京市
// 320102000000 玄武区
// <select name="province" data-value="32"></select>
// <select name="city" data-value="320100000000"></select>
// <select name="district" data-value="320102000000"></select>
// <select name="street" data-value=""></select>
// var area = new area_init(
// {
// province: "select[name='province']",
// city: "select[name='city']",
// district: "select[name='district']",
// street: "select[name='street']"
// }
// );
// 或者
// <select name="province" data-value="32"></select>
// <select name="city" data-value="320100000000"></select>
// var area = new area_init(
// {
// province: "select[name='province']",
// city: "select[name='city']"
// }
// );
// 如果需要js动态修改
// $('select[name="province"]').attr('data-value', data.province);
// $('select[name="city"]').attr('data-value', data.city);
// $('select[name="district"]').attr('data-value', data.district);
// area.load();
成品长这样哈
480
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
