利用scrapy爬取新浪体育上的图片

1、说明

我用的是python3下的scrapy，这篇博客主要是告诉大家如何用scrapy爬取图片并下载到本地。步骤我会一一说明

2、步骤

2、1 items部分

直接看代码
items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class SinaimageItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    ####图片链接
    image_urls = scrapy.Field()

2、2 spider部分

将spider部分命名为imagespider.py

#coding:utf-8  
import scrapy  
from sinaimage.items import SinaimageItem 

from scrapy.crawler import CrawlerProcess  
import re

class jiandanSpider(scrapy.Spider):  
    name = 'sina'  
    allowed_domains = []  
    start_urls = ["http://www.sina.com.cn/"]  

    def parse(self, response):  
        item = SinaimageItem()  
        item['image_urls'] = response.xpath('//img//@src').re(r'http.*?.jpg$')
        # print 'image_urls',item['image_urls']  
        yield item  
        new_url= response.xpath('//a[@class="uni-blk-pic"]//@href').extract()#翻页  
        # print 'new_url',new_url  
        for url in newurl:

            yield scrapy.Request(new_url,callback=self.parse)  

2、3 pipe部分

命名为 pipelines.py

import scrapy
import os 
import urllib 
from sinaimage import settings 

class MyImagesPipeline(object):
    def process_item(self, item, spider): 
        times = 0

        for image_url in item['image_urls']:
            times += 1
            path = 'E:/myimage/%d.jpg' % times
            w = open(path,'wb')
            coon = urllib.request.urlopen(image_url)
            w.write(coon.read())
            w.close()

        return item

2、4 setting部分

# -*- coding: utf-8 -*-

# Scrapy settings for sinaimage project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
#     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'sinaimage'

SPIDER_MODULES = ['sinaimage.spiders']
NEWSPIDER_MODULE = 'sinaimage.spiders'


ITEM_PIPELINES = {  
   'sinaimage.pipelines.MyImagesPipeline': 1,  
}  
DOWNLOAD_DELAY = 0.25  
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'sinaimage (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = True

3 结果