python爬取景点信息_Python用Pyspider爬取TripAdvisor的景点信息

先上效果图

上代码：from pyspider.libs.base_handler import *

import pymongo

class Handler(BaseHandler):

crawl_config = {

}

client = pymongo.MongoClient('localhost')

db = client['TripAdvise']

@every(minutes=24 * 60)

def on_start(self):

self.crawl('https://www.tripadvisor.cn/Attractions-g186338-Activities-c47-London_England.html',callback=self.index_page, validate_cert=False)

@config(age=10 * 24 * 60 * 60)

def index_page(self, response):

for each in response.doc('div.listing_title > a').items():

self.crawl(each.attr.href, callback=self.detail_page, validate_cert=False)

# 翻页

next = response.doc('.pagination .nav.next').attr.href

self.crawl(next, callback=self.index_page, validate_cert=False)

@config(priority=2)

def detail_page(self, response):

url = response.url

name = response.doc('.h1').text()

rating = response.doc(' div.ratingContainer > a > span').text()

garde = response.doc('div.section.rating > span').text()

address = response.doc('.contactInfo > .address').text()

phone = response.doc('div.contact > div.contactType.phone.is-hidden-mobile > div').text()

opening = response.doc('div.prw_rup.prw_common_atf_header_bl_responsive.headerBL > div > span').text()

introduction = response.doc('.centerWell > div > div > div > div > div > span').text()

return {

"url": url,

"name": name,

"rating": rating,

'garde': garde,

"address": address,

"phone": phone,

"opening": opening,

"introduction": introduction

}

def on_result(self, result):

if result:

self.save_to_mongo(result)

def save_to_mongo(self, result):

if self.db['london'].insert(result):

print('保存到MongoDB成功', result)