python网络爬虫文献综述_Python网络爬虫教程：知乎爬虫案例

#!/usr/bin/env python#-*- coding:utf-8 -*-

from scrapy.contrib.spiders importCrawlSpider, Rulefrom scrapy.selector importSelectorfrom scrapy.contrib.linkextractors.sgml importSgmlLinkExtractorfrom scrapy.http importRequest, FormRequestfrom zhihu.items importZhihuItemclassZhihuSipder(CrawlSpider) :

name= "zhihu"allowed_domains= ["www.zhihu.com"]

start_urls= [ "http://www.zhihu.com"]

rules=(

Rule(SgmlLinkExtractor(allow= ('/question/d+#.*?', )), ca

llback= 'parse_page', follow =True),

Rule(SgmlLinkExtractor(allow= ('/question/d+', )), callba

ck= 'parse_page', follow =True),

)

headers={"Accept": "*/*","Accept-Encoding": "gzip,deflate","Accept-Language": "en-US,en;q=0.8,zh-TW;q=0.6,zh;q=0.4","Connection": "keep-alive","Content-Type":"application/x-www-form-urlencoded; charset=UTF

-8","User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1)

AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/

537.36","Referer": "http://www.zhihu.com/" }#重写了爬⾍类的⽅法, 实现了⾃定义请求, 运⾏成功后会调⽤callback 回调

函数defstart_requests(self):return [Request("https://www.zhihu.com/login", meta = {'coo

kiejar': 1}, callback = self.post_login)]

#FormRequeset 出问题了

defpost_login(self, response):print 'Preparing login'

#下⾯这句话⽤于抓取请求⽹⻚后返回⽹⻚中的_xsrf 字段的⽂字, ⽤于成

功提交表单

xsrf= Selector(response).xpath('//input[@name="_xsrf"]/@va

lue').extract()[0]

printxsrf#FormRequeset.from_response 是 Scrapy 提供的⼀个函数, ⽤于 post 表 单#登陆成功后, 会调⽤after_login 回调函数

return [FormRequest.from_response(response, #"http://www.

zhihu.com/login",

okiejar']},

ers

meta= {'cookiejar' : response.meta['co

headers = self.headers, #注意此处的

head

formdata={'_xsrf': xsrf,'email': '1095511864@qq.com','password': '123456'},

callback=self.after_login,

dont_filter=True

)]defafter_login(self, response) :for url inself.start_urls :yieldself.make_requests_from_url(url)defparse_page(self, response):

problem=Selector(response)

item=ZhihuItem()

item['url'] =response.url

item['name'] = problem.xpath('//span[@class="name"]/text()').extract()print item['name']

item['title'] = problem.xpath('//h2[@class="zm-item-title zm-editable-content"]/text()').extract()

item['description'] = problem.xpath('//div[@class="zm-editable-content"]/text()').extract()

item['answer']= problem.xpath('//div[@class=" zm-editable-c

ontent clearfix"]/text()').extract()

return item