本次只爬取搜索的python相关的所有书籍
scrapy start project ddbook
(cd /ddbook/ddbook)
scrapy genspider -t basic book dangdang.com
然后打开 book.py
# http://search.dangdang.com/?key=Python&act=input&page_index=1#J_tab
# http://search.dangdang.com/?key=Python&act=input&page_index=2#J_tab
#一共100页
# -*- coding: utf-8 -*-
import scrapy
from ..items import DdbookItem
from scrapy.http import Request
import re
class BookSpider(scrapy.Spider):
name = 'book'
allowed_domains = ['dangdang.com']
header = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"}
# start_urls = ['http://dangdang.com/']
def start_requests(self):
for i in range(1, 2):
# for i in range(1, 101): #共100页
url = "http://search.dangdang.com/?key=Python&act=input&page_index=" + str(i) + "#J_tab"
print(url)
yield Request(url, callback=self.parse)
def parse(self, response):
for j in range(1,61): #每页60个
try:
item = DdbookItem()
author = "//ul[@class='bigimg']/li[@class='line"+str(j)+"']"
print(author)
if response.xpath(author+"//a/@title"):
item["title"] = response.xpath(author+"//a/@title")[0].extract()
print(item["title"])
else:
item["title"] = ''
if response.xpath(author+"//a[@name='itemlist-author']/text()"):
item["author"] = response.xpath(author+"//a[@name='itemlist-author']/text()")[0].extract()
print(item["author"])
else:
item["author"] = ''
if response.xpath(author + "//span[@class='search_now_price']/text()"):
item["price"] = response.xpath(author + "//span[@class='search_now_price']/text()")[0].extract()
print(item["price"])
else:
item["price"] = ''
if response.xpath(author + "//a[@name='P_cbs']/text()"):
item["press"] = response.xpath(author + "//a[@name='P_cbs']/text()")[0].extract()
print(item["press"])
else:
item["press"] = ''
if response.xpath(author + "//span/text()"):
alldata = response.xpath(author + "//span/text()").extract()
# print(alldata)
# a = len(alldata)
data = alldata[len(alldata)-2] #因为日期的位置从头数不一定是第七个,但一定是span的最后一个,又因为数组开头是0
# print(data)
# 因为出来的日期前面有个斜杠
pat = "\d{4}-\d{2}-\d{2}"
item["data"] = re.compile(pat).findall(data)[0] #[0]是为了去掉"
print(item["data"])
else:
data = ''
yield item
except Exception as e:
print(e)