python3使用scrapy爬取图片示例

安装scrapy

pip install scrapy -i https://pypi.tuna.tsinghua.edu.cn/simple 

创建项目

打开pycharm,进入终端 输入
scrapy startproject example
cd example
scrapy genspider image pic.netbian.com

项目结构

image—自己定义保存图片位置
image.py—爬虫文件
item.py----定义结构化数据字段
middlewares.py–中间件
pipelines.py—数据管道
settings.py–爬虫设置

具体实现

image.py

import time
import urllib.request

import scrapy
from ..items import GetimageItem
from scrapy.http import Request
import re


class ImageSpider(scrapy.Spider):
    name = 'image'
    allowed_domains = ['pic.netbian.com']
    start_urls = ['https://pic.netbian.com/4kmeinv/'] #此处为爬取的类型标题链接

    def parse(self, response, **kwargs):
        item = GetimageItem()
        pat = response.xpath('//ul[@class="clearfix"]/li/a/@href').extract()
        for i in range(len(pat)):
            new_url = "https://pic.netbian.com" + str(pat[i])
            data = urllib.request.urlopen(new_url).read().decode('gbk')
            pat_url = "src=\"(/uploads/allimg/.*?).jpg\""
            img_url = re.compile(pat_url).findall(data)
            item['url'] = img_url[0]
            time.sleep(1.5)
            yield item
        for i in range(2, 5):#此处显示爬取多少页
            url = "https://pic.netbian.com/4kmeinv/index_" + str(i) + ".html"
            time.sleep(1.5)
            yield Request(url, self.parse)

        # pass

item.py

import scrapy


class GetimageItem(scrapy.Item):
    # define the fields for your item here like:
    url = scrapy.Field()
    # pass

pipelines.py

import time
import urllib.request

from itemadapter import ItemAdapter


class GetimagePipeline:

    def process_item(self, item, spider):
        # for i in range(len(item['url'])):
        this_img = item['url']
        url = ("https://pic.netbian.com" + this_img + '.jpg')
        local_path = "E:\\pythonProject\\crawler\\getimage\\getimage\\image\\" + str(
            int(time.time() * 1000)) + '.jpg'  # 注：地址以自身文件夹地址为准
        urllib.request.urlretrieve(url, local_path)
        # time.sleep(1.5)
        return item

settings.py

# Scrapy settings for getimage project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'getimage'

SPIDER_MODULES = ['getimage.spiders']
NEWSPIDER_MODULE = 'getimage.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
# USER_AGENT = 'getimage (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = True

# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
# DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
# CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
# TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
    "Referer": "https://pic.netbian.com/4kmeinv/",
    "cache-control": " max-age=0",
    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
    "accept-encoding": "gzip, deflate, br",
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.134 Safari/537.36 Edg/103.0.1264.71"
}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
# SPIDER_MIDDLEWARES = {
#    'getimage.middlewares.GetimageSpiderMiddleware': 543,
# }

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = {
#    'getimage.middlewares.GetimageDownloaderMiddleware': 543,
# }

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
# EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
# }

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    'getimage.pipelines.GetimagePipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
# AUTOTHROTTLE_ENABLED = True
# The initial download delay
# AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
# AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
# AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
# HTTPCACHE_ENABLED = True
# HTTPCACHE_EXPIRATION_SECS = 0
# HTTPCACHE_DIR = 'httpcache'
# HTTPCACHE_IGNORE_HTTP_CODES = []
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
IMAGES_STORE = 'E:\\pythonProject\\crawler\\getimage\\getimage\\image\\'

运行

可在爬虫文件夹同目录下（即scrapy.cfg同位置）新建main.py文件，后使用pycharm自带运行工具运行即可

from scrapy import cmdline

cmdline.execute('scrapy crawl image'.split())  # 注：这里的image为爬虫名称即image.py文件中name

pycharm下载链接: https://www.jetbrains.com/pycharm/download/#section=windows.
python下载链接: https://www.python.org/downloads/
pycharm使用请参考：https://zhuanlan.zhihu.com/p/346316776

注：不可商用，仅供学习使用（产生侵权请与本人联系）