爬虫1.1爬取斗图啦图片（关于open函数和urlretrieve函数）

文章只是我作为NewBird     ٩꒰▽ ꒱۶⁼³₌₃     学习的一小点小点的进步

还请不要笑我⁄(⁄ ⁄•⁄ω⁄•⁄ ⁄)⁄

我就直接贴代码了，我不会说很技术的话。

1.创建项目命令：

scrapy startproject <project_name>

例子：

scrapy startproject myproject

创建成功如下图：

文件夹目录如下

1 2 3 4 5 6 7 8 9 10

myproject/     scrapy.cfg          -------项目的配置文件     myproject/         -------该项目的python模块。之后您将在此加入代码。         __init__.py         items.py       --------项目中的item文件.         pipelines.py  --------项目中的pipelines文件         settings.py    --------项目的设置文件         spiders/        --------放置spider代码的目录             __init__.py             ...

以上复制了别人的https://www.cnblogs.com/pachongshou/p/6125858.html

2.明确目标——Item.py

import scrapy

class Doutu2Item(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    img_url=scrapy.Field()
    name=scrapy.Field()

3.制作爬虫，先爬再取——spider.py

# -*- coding: utf-8 -*-
import scrapy
from ..items import Doutu2Item
from pyquery import PyQuery as pq
import os
import requests
from  urllib import request
import re
class DoutuSpider(scrapy.Spider):
    name = 'doutu'
    allowed_domains = ['doutula.com']

    start_urls = ['http://doutula.com/photo/list/?page={}'.format(i)for i in range(1,3)]

    def parse(self, response):
        jpy=pq(response.text)
        #我这里使用了PyQuery
        Zurl=jpy('#pic-detail > div > div.col-sm-9 > div.random_picture > ul > li > div > div>a').items()
        i=0
        for it in Zurl:
            #遍历Zurl
            print(it.text())
            #实例化item对象，进行存储
            item=Doutu2Item()
            #PyQuery中获取属性attr()
            #以下是动图和jpg的url获取
            item['img_url']=it('img').attr('data-original')
            item['name']=it('p').text()
            if  not item['img_url']:
                item['img_url']=it('img').eq(1).attr('data-original')
            print(item['img_url'])
            i+=1
            # if os.path.exists('斗图'):
            #     print('文件夹已存在')
            # else:
            #     os.makedirs('斗图')
            #     print('文件夹已经创建')
            if not os.path.exists('doutu'):
                print('创建文件夹:{}'.format('doutu'))
                os.mkdir('doutu')
            if not os.path.exists('pic'):
                print('创建文件夹:{}'.format('pic'))
                os.mkdir('pic')
            #正则表达式替换名称中有特殊字符的
            rstr = r"[\/\\\:\*\?\"\<\>\|]"  # '/ \ : * ? " < > |'
            new_title = re.sub(rstr, "_", item['name'])  # 替换为下划线

            #第一种存储方式：打开文件路径的时候我不太会弄，所以错了几次，相对路径比较好，
            with open('pic/%s.jpg' % new_title,'wb') as f:
                f.write(requests.get(item['img_url']).content)
            #第二种存储方式
            try:
                request.urlretrieve(item['img_url'],'doutu\%s.gif'% new_title)
            except:
                pass
            print(i)
            print('__________________________________________________')
        yield  item




4.处理spider抽取的item——pipeline.py


from scrapy.exceptions import DropItem
from scrapy import log
import json
from pymongo import MongoClient
from  scrapy.conf import settings


class Doutu2Pipeline(object):
	#这里是初始化方法
	#进行数据库的一些设置
    def __init__(self):
        connection=MongoClient(
            settings['MONGODB_SERVER'],
            settings['MONGODB_PORT']
        )
        db=connection[settings['MONGODB_DB']]
        self.collection=db[settings['MONGODB_COLLECTION']]

    def process_item(self, item, spider):
        self.collection.insert(dict(item))
        print("我已进入数据库")
        valid=True
        for data in item:
            if not data:
                valid=False
                raise DropItem('MIssing{}'.format(data))
        if valid:
            log.msg('已经进入数据库',level=log.DEBUG,spider=spider)


        return item
__init__方法在类的一个对象被建立时，马上运行。这个方法可以用来对你的对象做一些你希望的 初始化 。注意，这个名称的开始和结尾都是双下划线






5.配置文件——setting.py


# Scrapy settings for doutu2 project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
#     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'doutu2'
SPIDER_MODULES = ['doutu2.spiders']
NEWSPIDER_MODULE = 'doutu2.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 0.2
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
#}

# Enable or disable spider middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
SPIDER_MIDDLEWARES = {
   'doutu2.middlewares.Doutu2SpiderMiddleware': 543,
}

# Enable or disable downloader middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'doutu2.middlewares.MyCustomDownloaderMiddleware': 543,
#}

# Enable or disable extensions
# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'doutu2.pipelines.Doutu2Pipeline': 300,
}
MONGODB_SERVER = "localhost"
MONGODB_PORT = 27017
MONGODB_DB = "test"
MONGODB_COLLECTION = "doutu2"





6.中间件进行request和response一些设置——middlewares.py

        这个暂时还不怎么U