scrapy 图片下载
使用scrapy 下载图片
代码示例
spider.py
import scrapy
from DownloadPhoto.items import DownloadphotoItem
class TupianSpider(scrapy.Spider):
name = 'tupian'
allowed_domains = ['daimg.com']
start_urls = ['http://www.daimg.com/photo/']
def parse(self, response):
item = DownloadphotoItem()
for info in response.xpath("//ul[@class='ibox2_list']/li"):
item['image_urls'] = info.xpath(".//a/img/@src").extract()#这里传过去的必须是列表,不能是字符串
item['image_name'] = info.xpath(".//a/text()").extract_first()
yield item
spider.py主要抓取图片的名字或者链接然后yield这里不做过多讲解
setting.py
ITEM_PIPELINES = {
'DownloadPhoto.pipelines.ImagePipeline':3 #开启自定义的图片下载管道
# 'scrapy.pipelines.images.ImagesPipeline':1
}
#名字不能写错
IMAGES_STORE="image" #下载图片的地址
IMAGE_URLS_FILED="image_urls"#需要下载的图片url字段名 item中定义的名字
setting.py 开启管道 设置下载地址及需要下载的图片字段名
pipeline.py
import re
from scrapy.pipelines.images import ImagesPipeline
from scrapy import Request
class ImagePipeline(ImagesPipeline):
def get_media_requests(self, item, info):
for image_url in item['image_urls']:
yield Request(image_url,meta={"name":item['image_name']})
# 重命名,若不重写这函数,图片名为哈希,就是一串乱七八糟的名字
def file_path(self, request, response=None, info=None):
"""
分文件夹下载
image_name=request.meta['name']+".jpg"
filename=request.meta['name']
# 过滤windows字符串,不经过这么一个步骤,你会发现有乱码或无法下载
name=re.sub(r'[?\\*|“<>:/]', '', filename)
filename = u'{0}/{1}'.format(name, image_name)
return filename
"""
#上面函数传过来的
image_name=request.meta['name']
filename=request.meta['name']
# 过滤windows字符串,不经过这么一个步骤,你会发现有乱码或无法下载
name=re.sub(r'[?\\*|“<>:/]', '', filename)+".jpg"
#{0}匹配的是文件夹的名字 {1}是图片名
return name