边学边记,记录遇到的坑达成的小目标
scrapy下载图片移步scrapy下载项目图片
items:
import scrapy
class PictureItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
image_urls = scrapy.Field()
images = scrapy.Field()
image_paths = scrapy.Field()
pipelines:
import scrapy
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem
class PicturePipeline(object):
def get_media_requests(self, item, info):
for image_url in item['image_urls']:
yield scrapy.Request(image_url)
def item_completed(self, results, item, info):
image_paths = [x['path'] for ok, x in results if ok]
if not image_paths:
raise DropItem("Item contains no images")
item['image_paths'] = image_paths
return item
settings:添加想要保存图片的路径
IMAGES_STORE = 'H:\\new\\pic'
ITEM_PIPELINES = {
'Picture.pipelines.PicturePipeline': 300,
'scrapy.pipelines.images.ImagesPipeline': 1
}
spider:
# -*- coding: utf-8 -*-
import scrapy
from Picture.items import PictureItem
class PictureSpider(scrapy.Spider):
name = 'spider名字'
allowed_domains = ['想要爬取的网站']
start_urls = ['想要爬取的网站']
def parse(self, response):
item=PictureItem()
item['image_urls']=response.xpath(".//div[@class='thumb']/a/img/@src").extract()
next_page=response.xpath(".//div[@class='pagebar']/a[contains(text(),'下一页')]/@href").extract()[0]
yield scrapy.Request(next_page,callback=self.parse)
yield item
gif再研究研究