无聊中看到php中curl模块可以抓取数据,简单实现以下:
需求分析:
抓取大众点评数据
住区内容
1,地区
2,分类
3,店铺详细信息,店铺名称,店铺招牌,地址, 电话, 营业时间,人均消费,其他分店(关联其他分店),环境图片
http://m.dianping.com/citylist
1,定义的简单的curl类库:
<?php
namespace getdp;
class CURL {
private $ch;
private $flag_if_have_run;
public function __construct($url) {
$this->ch = curl_init($url);
curl_setopt($this->ch, CURLOPT_RETURNTRANSFER , 1 );
}
public function close() {
curl_close($this->ch);
}
public function __destruct() {
$this->close();
}
public function set_time_out($timeout) {
curl_setopt($this->ch, CURLOPT_TIMEOUT, intval($timeout));
return $this;
}
public function set_referer($referer) {
if (!empty($referer))
curl_setopt($this->ch, CURLOPT_REFERER , $referer);
return $this;
}
public function load_cookie($cookie_file) {
curl_setopt($this->ch, CURLOPT_COOKIEFILE , $cookie_file);
return $this;
}
public function save_cookie($cookie_file="") {
if(empty($cookie_file))
$cookie_file = tempnam('./', 'cookie');
curl_setopt($this->ch, CURLOPT_COOKIEJAR , $cookie_file);
return $this;
}
public function exec () {
$str = curl_exec($this->ch);
$this->flag_if_have_run = true;
return $str;
}
public function post ($post) {
curl_setopt($this->ch, CURLOPT_POST , 1);
curl_setopt($this->ch, CURLOPT_POSTFIELDS , $post );
return $this;
}
public function get_info() {
if($this->flag_if_have_run == true )
return curl_getinfo($this->ch);
else
throw new Exception("aaaaa");
}
public function set_proxy($proxy) {
curl_setopt($this->ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
curl_setopt($this->ch, CURLOPT_PROXY,$proxy);
return $this;
}
public function set_ip($ip) {
if(!empty($ip))
curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("X-FORWARDED-FOR:$ip", "CLIENT-IP:$ip"));
return $ip;
}
public function set_browser($user_agent, $language) {
curl_setopt ($this->ch , CURLOPT_HTTPHEADER, array ("User-Agent: $user_agent","Accept-Language: $language"));
return $this;
}
}
2,使用pdo操作数据库,将解析获取的数据插入数据库
<?php
require 'curl.class.php';
$pdo = new \PDO('mysql:host=localhost;dbname=getdazong', 'root', '');
$pdo->query("set names utf8");
require 'functions.php';
3,functions.php,简单的pdo数据库操作方法
<?php
function getCitys() {
global $pdo;
return $pdo->query("select * from city")->fetchAll();
}
function getCityById($id) {
global $pdo;
return $pdo->query("select * from city where id = $id")->fetch();
}
function getShops(){
global $pdo;
return $pdo->query("select id from shop group by id")->fetchAll();
}
创建的数据库表:
1,city
CREATE TABLE `city` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`city` varchar(64) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=2506 DEFAULT CHARSET=utf8
2,category
CREATE TABLE `category` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`pid` int(11) NOT NULL,
`name` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=32743 DEFAULT CHARSET=utf8
3,店铺表
CREATE TABLE `shop` (
`id` int(32) unsigned NOT NULL,
`name` varchar(255) NOT NULL COMMENT '店铺名称',
`subname` varchar(255) DEFAULT NULL COMMENT '分店名称',
`area` varchar(64) NOT NULL COMMENT '店铺区域',
`address` varchar(255) DEFAULT NULL COMMENT '店铺地址',
`mobile` varchar(32) DEFAULT NULL COMMENT '联系电话',
`per_consumption` varchar(12) DEFAULT NULL COMMENT '消费',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
4,店铺环境图片表
CREATE TABLE `shop_image` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT,
`shop_id` int(11) NOT NULL,
`image_path` varchar(255) NOT NULL,
`type` tinyint(1) NOT NULL DEFAULT '0' COMMENT '1:店铺招牌,0,店铺环境图片',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=526 DEFAULT CHARSET=utf8 CHECKSUM=1 DELAY_KEY_WRITE=1 ROW_FORMAT=DYNAMIC
现在我们开始抓取数据啦:
1:抓取http://m.dianping.com/citylist所有城市数据
<?php
require 'common.php';
$homePage = "http://m.dianping.com/citylist";
$curl = new \getdp\CURL($homePage);
$homePageContent = $curl->exec();
$hrefRegex = "/<a οnclick=\"_gaq\.push\(.*\)\" href=\"javascript:window.location.href='\/c([0-9]{1,})\.csf'\" title=\".*\">(.*)<\/a>/i";
preg_match_all($hrefRegex, $homePageContent, $hrefs);
foreach ($hrefs[0] as $k => $v) {
$pdo->query("insert into city values (" . $hrefs[1][$k] . ", " . "'". $hrefs[2][$k] . "') on duplicate key update id = id");
}
2,获取店铺种类的数据
<?php
set_time_limit(0);
require 'common.php';
$url = "http://m.dianping.com";
$curl = new \getdp\CURL($url);
$curl->save_cookie('./cookie.txt')->load_cookie('./cookie.txt')->set_time_out('30')->exec();
for ($i = 1; $i <= 30000; $i++) {
$url = "http://m.dianping.com/getchildrencategory?categoryid=$i";
$curl = new \getdp\CURL($url);
$content = $curl->save_cookie('./cookie.txt')->load_cookie('./cookie.txt')->exec();
$content = json_decode($content, true);
$message = $content['message'];
if (!empty($message['category']) && is_array($message['category'])) {
foreach ($message['category'] as $category) {
$pdo->query("insert into category values ({$category['categoryId']}, $i, '{$category['categoryName']}') on duplicate key update id = id");
}
}
}
3,获取所有地区的店铺基本信息
<?php
set_time_limit(0);
require 'common.php';
$url = "http://m.dianping.com";
$curl = new \getdp\CURL($url);
$curl->save_cookie('./cookie.txt')->load_cookie('./cookie.txt')->set_time_out('30')->exec();
$cityList = getCitys();
foreach ($cityList as $item){
$index = 1;
while(true){
$url = "http://m.dianping.com/shoplist/{$item['id']}?reqType=ajax&page=$index";
$curl = new \getdp\CURL($url);
$homePageContent = $curl->save_cookie('./cookie.txt')->load_cookie('./cookie.txt')->set_time_out('30')->exec();
$hrefRegex = "/href=\"\/shop\/(.*)\">.*<div class=\"intro Fix\">.*<span>(.*)<\/span>/Us";
preg_match_all($hrefRegex, $homePageContent, $hrefs);
foreach ($hrefs[0] as $k => $v) {
//get shop basic information
$url = "http://m.dianping.com/shop/".$hrefs[1][$k];
$area = $hrefs[2][$k];
$curl = new \getdp\CURL($url);
$homePageContent = $curl->save_cookie('./cookie.txt')->load_cookie('./cookie.txt')->set_time_out('30')->exec();
$regex ="/<div class=\"details-mode shop-info\">.*<img src=\"(.*)\">.*<span class=\"shop-name\">(.*)<\/span>.*<span class=\"price\">.*[人均|消费|费用]:(.*)<\/span>/Us";
preg_match($regex, $homePageContent,$matches);
$sign_image = !empty($matches[1]) ? $matches[1]:'';
$temp_name = !empty($matches[2]) ? $matches[2] :'';
$per_consumption = !empty($matches[3]) ? trim($matches[3]):'';
$regex = "/(.*)\((.*)\)/Us";
preg_match($regex, $temp_name,$temp_matches);
if(count($temp_matches)>0){
$name = $temp_matches[1];
$subname = $temp_matches[2];
}else{
$name = $temp_name;
$subname = '';
}
$regex = "/<i class=\"icon-address\"><\/i>(.*)<i class=\"arrowent\"><\/i>.*href=\"tel:(.*)\"/Us";
preg_match($regex, $homePageContent,$_matches);
$address = !empty($_matches[1])?$_matches[1]:'';
$mobile = !empty($_matches[2])? $_matches[2]:'';
$pdo->query("INSERT INTO shop(id,name,subname,area,address,mobile,per_consumption) VALUES(".$hrefs[1][$k].",'".$name."','".$subname."','".$area."','".$address."','".$mobile."','".$per_consumption."')");
$pdo->query("INSERT INTO shop_image(shop_id,image_path,TYPE) VALUES(".$hrefs[1][$k].",'".$sign_image."',1)");
}
if(count($hrefs[0])<25) break;
$index++;
}
}
4,获取所有店铺的环境图片
<?php
set_time_limit(0);
require 'common.php';
$url = "http://m.dianping.com";
$curl = new \getdp\CURL($url);
$curl->save_cookie('./cookie.txt')->load_cookie('./cookie.txt')->set_time_out('30')->exec();
$shopList = getShops();
foreach ($shopList as $item){
$existedCityids = file_get_contents('./a.txt');
if(is_numeric(strpos($existedCityids,$item['id']))) continue;
$index = 1;
while (true) {
$url = "http://m.dianping.com/shop/{$item['id']}/photos?reqType=ajax&page=$index";
$curl = new \getdp\CURL($url);
$homePageContent = $curl->save_cookie('./cookie.txt')->load_cookie('./cookie.txt')->set_time_out('30')->exec();
$regex= "/<img src=\"(.*)\" οnerrοr=\"DP\.prior\.nofind\(\)\">/";
preg_match_all($regex, $homePageContent, $matches);
foreach ($matches[0] as $k => $v){
$pdo->query("INSERT INTO shop_image(shop_id,image_path,TYPE) VALUES(".$item['id'].",'".$matches[1][$k]."',0)");
}
if(count($matches[0])<15) break;
$index++;
}
file_put_contents('./a.txt', $item['id']."\n",FILE_APPEND);
}