这里通过pyspider框架
可以直接:pip3 install pyspider 下载框架
pyspider all 运行 可以看到 run 0.0.0.0:5000
直接在浏览器输入localhost:5000 进入 创建新项目
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2018-08-03 09:59:05
# Project: TripAdcisor
from pyspider.libs.base_handler import *
#import ssl
import pymongo
#ssl._create_default_https_context=ssl._create_unverified_context
class Handler(BaseHandler):
crawl_config = {
}
conn = pymongo.MongoClient('localhost')
db = conn.trip
myset = db.london
@every(minutes=24 * 60)
#on_start 启动目标主网站,validate_cert = False跳过证书检测,callback回调函数
def on_start(self):
self.crawl('https://www.tripadvisor.cn/Attractions-g186338-Activities-c47-London_England.html', callback=self.index_page,validate_cert = False)
@config(age=10 * 24 * 60 * 60)
#通过内置pyquery 获取目标网页的连接遍历 分别访问,并回调
def index_page(self, response):
for each in response.doc('.listing_title > a').items():
self.crawl(each.attr.href, callback=self.detail_page,validate_cert = False)
#获取下一页链接地址访问并回调自身
next = response.doc('#FILTERED_LIST > div.al_border.deckTools.btm > div > div > a').attr.href
print(next)
self.crawl(next,callback= self.index_page,validate_cert=False)
@config(priority=2)
#获得每个链接的的详细信息返回
def detail_page(self, response):
url = response.url
name = response.doc('.heading_title').text()
rating = response.doc('div.rating > .more').text()
location = response.doc('#taplc_attraction_detail_listing_0 > div.section.location > div.detail_section.address').text()[2:]
phone = response.doc('.phone > div').text()
durtion = response.doc('.hours > .duration').text()
return {
'name':name,
'rating':rating,
'location':location,
'phone':phone,
'durtion':durtion,
'url':url
}
#重写on_result方法 保存至数据库
def on_result(self,result):
if result:
self.save_to_mongo(result)
def save_to_mongo(self,result):
if self.myset.insert(result):
print('save ok!',result)
运行时将status 调至running 或者debug run即可