scrapy 的简单应用 爬取图片之家

scrapy 2.3版本

目的：https://www.tupianzj.com/meinv/ 网站的图片爬取

gitHUB:https://github.com/fddqfddq/scrapy/tree/master/tupianzj

1.创建项目

scrapy startproject tupianzjproject

2.创建crawl,使用crawl 模板创建

 scrapy genspider tupianzj tupianzj.com -t crawl  

3.修改items.py

url:网站url

category:类别

title:标题

imgurl:图片链接

imgname:图片标题

updatetime:更新时间

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy


class TupianzjItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    
    url = scrapy.Field()
    category = scrapy.Field()
    title = scrapy.Field()
    imgurl = scrapy.Field()
    imgname = scrapy.Field()
    updatetime = scrapy.Field()

 

4.修改pipelines.py

from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem
from scrapy.http import Request
import re

class TupianzjPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        for image_url in item['imgurl']:
            yield Request(image_url,meta={'item':item['imgname']})

    def file_path(self, request, response=None, info=None):
        name = request.meta['item']
        # name = filter(lambda x: x not in '()0123456789', name)
        name = re.sub(r'[？\\*|“<>:/()0123456789]', '', name)
        image_guid = request.url.split('/')[-1]
        # name2 = request.url.split('/')[-2]
        filename = u'full/{0}/{1}'.format(name, image_guid)
        return filename
        # return 'full/%s' % (image_guid)

    def item_completed(self, results, item, info):
        # image_path = [x['path'] for ok, x in results if ok]
        # if not image_path:
        #     raise DropItem('Item contains no images')
        # item['image_paths'] = image_path
        return item
    

5.修改spiders下的tupianzj.py

  注意的是：item['imgurl'] = response.xpath("//div[@id='bigpic']//img//@src").getall()# 这里要用getall()，否则会报错，div中id为bigpic下的图片的src

#爬取页面以.html为结尾的链接，因为详细页面是一样的，所以可以通用爬取，
#如果是子栏目页面如https://www.tupianzj.com/meinv/xingan ，有分页的也可以爬取。
#scrapy crawl tupianzj  
#scrapy crawl tupianzj -a tag=xingan    ,爬取https://www.tupianzj.com/meinv/xingan  
import scrapy
from tupianzj.items import TupianzjItem

class TupianzjSpider(scrapy.Spider):
    name = 'tupianzj'
    def start_requests(self):
        url = 'https://www.tupianzj.com/meinv/'
        tag = getattr(self, 'tag', None)
        if tag is not None:
           url = url + tag
        
        yield scrapy.Request(url, self.parse)

    def parse(self, response):
        #page_links = response.xpath("//div[@id='container']//a") #div中id为container下的所有超链接
        #page_links = response.xpath("//div[@class='warpbox_con']//a") #div中样式为warpbox_con下的所有超链接
        #page_links = response.css("ul.mv_list_r li a")
        #page_links = response.css("div.warpbox_con a[href$='.html']")
        page_links = response.css("a[href$='.html']") #页面所有的以.html为结尾的链接
        yield from response.follow_all(page_links, self.parse_detail)
        #next_link = response.css("div.pages ul li:nth-last-child(3) a::attr(href)").get() #div中的样式为pages,下的ul下的li下的超链接列表中的倒数第3个超链接的href
        next_link = response.css("div.pages ul li:nth-last-child(3) a") #div中的样式为pages,下的ul下的li下的超链接列表中的倒数第3个超链接
        if next_link is not None:
            yield from response.follow_all(next_link, self.parse)

    def parse_detail(self, response):
        try:
            if response.xpath("//div[@id='bigpic']//img//@src").get() is not None:
                item = TupianzjItem()
                item['url'] = response.url
                item['category'] = response.css("div.weizhi a:last-child::text").get(default='') #div中样式为weizhi下的最后一个超链接的文本
                item['title'] = response.xpath("//div[@class='warp']//h1//text()").get(default='') 
                item['imgurl'] = response.xpath("//div[@id='bigpic']//img//@src").getall()# 这里要用getall()，否则会报错，div中id为bigpic下的图片的src
                item['imgname'] = response.xpath("//div[@class='warp']//h1//text()").get(default='')
                item['updatetime'] = response.xpath("//div[@class='article_info']//u//text()").get(default='')
                yield item
                next_url = response.css("div.pages ul li:last-child a::attr(href)").get() #分页超链接
                if next_url is not None:
                    # 下一页
                    yield response.follow(next_url, callback=self.parse_detail)
        except Exception as e:
            print(e)
        finally:
            pass
            

        

6.修改settings.py

IMAGES_STORE = 'C:\D\ImagesRename' #图片下载的路径

DOWNLOADER_MIDDLEWARES = {

   'tupianzj.middlewares.TupianzjDownloaderMiddleware': 543,  # 图片下载中间件

   'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,

}

RANDOM_UA_TYPE = 'random'   #使用随机user-agent

# Enable or disable extensions

# See https://docs.scrapy.org/en/latest/topics/extensions.html

#EXTENSIONS = {

#    'scrapy.extensions.telnet.TelnetConsole': None,

#}

HTTPERROR_ALLOWED_CODES = [514]   #允许514返回code

 

# Scrapy settings for tupianzj project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'tupianzj'

SPIDER_MODULES = ['tupianzj.spiders']
NEWSPIDER_MODULE = 'tupianzj.spiders'

#save path
IMAGES_STORE = 'C:\D\ImagesRename' #图片下载的路径

# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'tupianzj (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = True

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
#}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'tupianzj.middlewares.TupianzjSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
   'tupianzj.middlewares.TupianzjDownloaderMiddleware': 543,  # 图片下载中间件
   'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
}
RANDOM_UA_TYPE = 'random'   #使用随机user-agent
# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}
HTTPERROR_ALLOWED_CODES = [514]   #允许514返回code
# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    'tupianzj.pipelines.TupianzjPipeline': 300,
    #'tupianzj.JsonWriterPipeline.JsonWriterPipeline': 200,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

7.修改middlewares.py

使用user-agent

# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals

# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter

from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
from fake_useragent import UserAgent
import random

class TupianzjSpiderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, or item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Request or item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class TupianzjDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.
    '''
    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        referer = request.url
        if referer:
            request.headers['referer'] = referer
    '''


    # def __init__(self, user_agent):
    #     self.user_agent = user_agent
 
    # @classmethod
    # def from_crawler(cls, crawler):
    #     return cls(
    #         user_agent=crawler.settings.get('USER_AGENT')
    #     )
 
    # def process_request(self, request, spider):
    #     referer = request.url
    #     if referer:
    #         request.headers['referer'] = referer
    #     agent = random.choice(self.user_agent)
    #     request.headers['User-Agent'] = agent

    #随机更换user-agent
    def __init__(self, crawler):
        super(TupianzjDownloaderMiddleware, self).__init__()
        self.ua = UserAgent()
        self.ua_type = crawler.settings.get("RANDOM_UA_TYPE", "random")

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler)

    def process_request(self, request, spider):
        def get_ua():
            return getattr(self.ua, self.ua_type)
        referer = request.url
        if referer:
            request.headers['referer'] = referer
        request.headers.setdefault('User-Agent', get_ua())

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)