pyspider 爬豆瓣电影信息

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2015-11-23 10:50:38
# Project: doubanmovie
from pyspider.libs.base_handler import *
import pymongo,sys
from pymongo import MongoClient
import re,time,random,hashlib
from scrapy.selector import Selector
from datetime import *
class Handler(BaseHandler):

client = MongoClient(host="localhost",port=27017, read_preference=pymongo.read_preferences.ReadPreference.PRIMARY_PREFERRED)
db = client.edogdata
db.authenticate("database","passwd")

@every(minutes=24*60)
def on_start(self):
self.crawl('http://movie.douban.com/tag/', callback=self.index_page) #豆瓣电影标签

@config(age=10 * 24 * 60 * 60)
def index_page(self, response):
for each in response.doc('a[href^="http"]').items():
if re.match("http://www.douban.com/tag/\w+", each.attr.href, re.U): # 爱情，戏剧，刘德华，1990，惊悚，恐怖 。。。
self.crawl(each.attr.href, callback=self.list_page)

@config(age=2, priority=2)
def list_page(self, response):
#print "urlllll response",response.doc
movie = response.doc('a[href^="http"]').items()
for each in movie:
if re.match("http://www.douban.com/link2/\W+",each.attr.href):
#print "each",each.text()
#print "each.attr.href",each.attr.href
if each.attr.href.find("movie") < 130:
self.crawl(each.attr.href, callback=self.final_page)
print "find movie"

def final_page(self,response):
for each in response.doc('a[href^="http"]').items():
if re.match('http://movie.douban.com/\w+',each.attr.href,re.U):
self.crawl(each.attr.href,callback=self.detail_page)
#翻页
self.crawl([x.attr.href for x in response.doc('.next a').items()],callback=self.final_page) # .next 的 '.' 表示class=next ，如果是 #next 的话，则表示 id=next ×××××××××××××××××××××××××××××××××××××××××××××××××××
#@config(priority=4)
def detail_page(self, response):
now = str(datetime.now())
_id = hashlib.md5(response.doc('HTML>BODY>DIV#wrapper>DIV#content>H1>SPAN').text().encode('utf-8')).hexdigest()
site = response.url
name = response.doc("title").text().split('(')[0]#response.doc('HTML>BODY>DIV#wrapper>DIV#content>H1>SPAN[property="v:itemreviewed"]').text()
director = [x.text() for x in response.doc('a[rel="v:directedBy"]').items()]
release_date = [x.text() for x in response.doc('span[property="v:initialReleaseDate"]').items()]
actor = '/'.join([x.text() for x in response.doc('a[rel="v:starring"]').items()])
rating = [x.text() for x in response.doc('strong[property="v:average"]').items()]
type = [x.text() for x in response.doc('span[property="v:genre"]').items()]
source = '豆瓣'
#IMDb_Link = [x.attr.href for x in response.doc('a[rel="nofollow"]').items()]
IMDb_Link = response.doc('HTML>BODY>DIV#wrapper>DIV#content>DIV.grid-16-8.clearfix>DIV.article>DIV.indent.clearfix>DIV.subjectwrap.clearfix>DIV.subject.clearfix>DIV#info>a[rel="nofollow"]').attr.href # HTML>BODY 可以不用
self.db.douban.insert({"_id":_id,"url":site , "title":name,"time":now,"director":director,"release_date": release_date,'actor':actor,"type": type,"source": source,"rating":rating,"IMDb_Link":IMDb_Link})
return {
"url": response.url,
"title": name,#response.doc('HTML>BODY>DIV#wrapper>DIV#content>H1>SPAN[property="v:itemreviewed"]').text(),#property="v:itemreviewed
"rating": rating,#response.doc('HTML>BODY>DIV#wrapper>DIV#content>DIV.grid-16-8.clearfix>DIV.article>DIV.indent.clearfix>DIV.subjectwrap.clearfix>DIV#interest_sectl>DIV.rating_wrap.clearbox>DIV.rating_self.clearfix>STRONG.ll.rating_num').text(),
"导演": actor,#[x.text() for x in response.doc('a[rel="v:directedBy"]').items()],
"time": now,
"release_date" : release_date,
"actor" : actor,
"type" : type,
"IMDb_Link" : IMDb_Link
}