爬取苏宁图书案例
1.项目文件架构
2.爬取数据网页页面
3.suning.py文件
# -*- coding: utf-8 -*-
import scrapy
from copy import deepcopy
import re
class SuningSpider(scrapy.Spider):
name = 'suning'
allowed_domains = ['suning.com']
start_urls = ['https://list.suning.com/1-502325-0.html']
def parse(self, response):
#大分类
li_list = response.xpath("//div[@id='filter-results']/ul/li")
# print(li_list)
for li in li_list:
item = {}
item["introduction"] = li.xpath(".//img[@class='search-loading']/@alt").extract_first()
item["img"] = li.xpath(".//div[@class='img-block']//img/@src2").extract_first()
item["img"] = "http:" + item["img"]
#价格取不到# item["price"] = li.xpath("//*[@id='filter-results']/ul/li[2]/div/div/div/div[2]/p[1]/em/b//text()").extract()
item["store_name"] = li.xpath(".//p[@class='seller oh no-more ']/@salesname").extract_first()
item["book_href"] = li.xpath(".//div[@class='img-block']/a/@href").extract_first()
item["book_href"] = "https:" + item["book_href"]
# yield scrapy.Request(
# item["book_href"],
# callback=self.get_price,
# meta = {"item":deepcopy(item)} #深拷贝防止并发覆盖
# )
# yield item #对应操作每一个信息都是一个全新的item字典空间,所以不需要考虑覆盖问题。
print(item)
# def get_price(self, response):
# pass
# 翻页
page_count = int(re.findall("param.pageNumbers = \"(.*?)\";", response.body.decode())[0])
current_page = int(re.findall("param.currentPage = \"(.*?)\";", response.body.decode())[0])
print("第" + str(current_page) + "页" + "*"*50)
if current_page < page_count:
print("执行到内部")
next_url = "https://list.suning.com/emall/showProductList.do?ci=502325&pg=03&cp={}".format(current_page + 1)
yield scrapy.Request(
next_url,
callback=self.parse,
# meta={"item": response.meta["item"]} #只传输纯净的小分类信息(洁癖)
# meta = {"item":item}
dont_filter=True
)
- 当存在读取大分类下小分类的情况,应使用深拷贝防止数据覆盖。
4.pipelines.py文件
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
class BookPipeline(object):
def process_item(self, item, spider):
print(item)
return item
- 此处没有什么东西,就是打印爬取的数据
5.settings.py文件
# -*- coding: utf-8 -*-
# Scrapy settings for book project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://doc.scrapy.org/en/latest/topics/settings.html
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'book'
SPIDER_MODULES = ['book.spiders']
NEWSPIDER_MODULE = 'book.spiders'
LOG_LEVEL ="WARNING"
# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False #不遵循不能爬虫请求
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'book.middlewares.BookSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'book.middlewares.BookDownloaderMiddleware': 543,
#}
# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'book.pipelines.BookPipeline': 300,
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
- 注意一定要将ROBOTSTXT_OBEY 设为False从下一页开始便不满足robots协议了。