一、爬取豆瓣9+评分书籍
http://www.bubuko.com/infodetail-1533174.html
在cmd里面直接创建scrapy工程,然后进入工程genspider一个爬虫工程。创建main函数,对settings 和 items进行设置
# -*- coding: utf-8 -*-
import scrapy
import re
from doubanbook.items import DoubanbookItem
class DbbookSpider(scrapy.Spider):
name = 'dbbook'
#allowed_domains = ['https://www.douban.com/doulist/1264675/']
start_urls = ['https://www.douban.com/doulist/1264675/']
def parse(self, response):
item = DoubanbookItem()
selector = scrapy.Selector(response)
books = selector.xpath('//div[@class="bd doulist-subject"]')
for each in books:
title = each.xpath('div[@class="title"]/a/text()').extract()[0]
rate = each.xpath('div[@class="rating"]/span[@class="rating_nums"]/text()').extract()[0]
author = re.search('<div class="abstract">(.*?)<br',each.extract(),re.S).group(1)
title = title.replace(' ','').replace('\n','')
author = author.replace(' ','').replace('\n','')
item['title'] = title
item['rate'] = rate
item['author'] = author.split(':')[1]
yield item
nextPage = selector.xpath('//span[@class="next"]/link/@href').extract()
if nextPage:
next = nextPage[0]
print(next)
yield scrapy.http.Request(next,callback=self.parse)
scrapy 的selectors 使用xpath 和 CSS机制提取网页内容
import scrapy
from tutorial.items import DmozItem
class DmozSpider(scrapy.Spider):
name = "dmoz"
allowed_domains = ["dmoz.org"]
start_urls = [
"http://www.dmoz.org/Computers/Programming/Languages/Python/",
]
def parse(self, response):
for href in response.css("ul.directory.dir-col > li > a::attr('href')"):
url = response.urljoin(response.url, href.extract())
yield scrapy.Request(url, callback=self.parse_dir_contents)
def parse_dir_contents(self, response):
for sel in response.xpath('//ul/li'):
item = DmozItem()
item['title'] = sel.xpath('a/text()').extract()
item['link'] = sel.xpath('a/@href').extract()
item['desc'] = sel.xpath('text()').extract()
yield item