scrapy 图片下载

scrapy 图片下载

使用scrapy 下载图片

代码示例

spider.py

import scrapy
from DownloadPhoto.items import DownloadphotoItem

class TupianSpider(scrapy.Spider):
    name = 'tupian'
    allowed_domains = ['daimg.com']
    start_urls = ['http://www.daimg.com/photo/']

    def parse(self, response):
        item = DownloadphotoItem()
        for info in response.xpath("//ul[@class='ibox2_list']/li"):
            item['image_urls'] = info.xpath(".//a/img/@src").extract()#这里传过去的必须是列表，不能是字符串
            item['image_name'] = info.xpath(".//a/text()").extract_first()
            yield item

spider.py主要抓取图片的名字或者链接然后yield这里不做过多讲解

setting.py

ITEM_PIPELINES = {
    'DownloadPhoto.pipelines.ImagePipeline':3 #开启自定义的图片下载管道
   # 'scrapy.pipelines.images.ImagesPipeline':1
}
#名字不能写错
IMAGES_STORE="image" #下载图片的地址 
IMAGE_URLS_FILED="image_urls"#需要下载的图片url字段名 item中定义的名字

setting.py 开启管道 设置下载地址及需要下载的图片字段名

pipeline.py

import re
from scrapy.pipelines.images import ImagesPipeline
from scrapy import Request


class ImagePipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        for image_url in item['image_urls']:
            yield Request(image_url,meta={"name":item['image_name']})

    # 重命名，若不重写这函数，图片名为哈希，就是一串乱七八糟的名字
    def file_path(self, request, response=None, info=None):
         """
        分文件夹下载
            image_name=request.meta['name']+".jpg"
            filename=request.meta['name']
            # 过滤windows字符串，不经过这么一个步骤，你会发现有乱码或无法下载
            name=re.sub(r'[？\\*|“<>:/]', '', filename)
            filename = u'{0}/{1}'.format(name, image_name)
            return filename
        """
        #上面函数传过来的
        image_name=request.meta['name']
        filename=request.meta['name']
        # 过滤windows字符串，不经过这么一个步骤，你会发现有乱码或无法下载
        name=re.sub(r'[？\\*|“<>:/]', '', filename)+".jpg"
        #{0}匹配的是文件夹的名字  {1}是图片名
        return name