异步I/O续和Scrapy

异步I/O续和Scrapy

死锁（deadlock）

• 死锁的四个条件
• 禁止抢占
• 持有和等待
• 互斥
• 循环等待

循环引用问题

• weakref 弱引用

"""
import weakref
class A():
    pass

class B():
    pass

a = A()
b = B()
# 在这里发生了循环调用，如果不做处理会发生严重的内存泄漏
# 所以导入了weakref 让其中的一方弱引用，这样就不会算到引用计数里面，垃圾回收机制就会把它释放
a.dept = b
b.mgr = weakref.ref(a)

"""

处理数据套路

• 几个比较重要的内置函数
• 第一步：filter 过滤（筛选需要的数据）
• 第二步：map 映射（把数据映射成想要的格式）
• 第三歩：reduce(python 中 sum / max / min) 规约（提取出有用的信息）
• 

import math
thy_list = [1, 2, 3, 4, 5, 6, 7, 8]
print(sum(map(math.sqrt, filter(lambda x: x % 2 == 0, thy_list))))

8.692130429902463

异步I/O

import asyncio
import time


@asyncio.coroutine
def countdown(name, num):
    while num > 0:
        print(f'Countdown[{name}]: {num}')
        # 异步执行 - 非阻塞
        yield from asyncio.sleep(1)
        # 同步执行 - 阻塞式的
        # time.sleep(1)
        num -= 1


def main():
    loop = asyncio.get_event_loop()
    # 异步I/O - 虽然只有一个线程但是两个任务相互之间不阻塞
    tasks = [
        countdown("A", 10), countdown("B", 5),
    ]
    loop.run_until_complete(asyncio.wait(tasks))
    loop.close()


if __name__ == '__main__':
    main()

import asyncio

# pip install aiohttp
# 用生成器请求网页的第三方库
import aiohttp


@asyncio.coroutine
async def download(url):
    # 异步的方式取数据
    print('[Fetch]:', url)
    # 建立会话
    async with aiohttp.ClientSession() as session:
        # 获取页面，与requests用法相似
        async with session.get(url) as resp:
            print(url, '--->', resp.status)
            print(url, '--->', resp.cookies)
            print('\n\n', await resp.text())


def main():
    loop = asyncio.get_event_loop()
    urls = [
        'https://www.baidu.com',
        'http://www.sohu.com',
        'http://www.sina.com',
        'https://www.taobao.com',
        'http://www.qq.com'
    ]
    tasks = [download(url) for url in urls]
    loop.run_until_complete(asyncio.wait(tasks))


if __name__ == '__main__':
    main()

Scrapy

• 创建虚拟环境
• python -m venv 虚拟环境名
• 创建一个内置原来环境中已有第三方库的环境
• python -m venv 虚拟环境名 –system-site-packages
• python -m pip install -U pip
• pip install scrapy
  
  • 如果报错
  • 需要安装这个依赖库Twisted-18.4.0-cp36-cp36m-win_amd64.whl
  • pip install 文件路径+Twisted-18.4.0-cp36-cp36m-win_amd64.whl
• 创建项目
• scrapy startproject 项目名
• 创建爬虫

• scrapy genspider 爬虫名字 域名

• scrapy shell url 用shell来试运行

  • pip install pypiwin32
• 启动
• scrapy crawl 爬虫文件名
• 将文件结果导入result.json

• scrapy crawl 爬虫文件名 -o result.json

• settings.py中

# -*- coding: utf-8 -*-

# Scrapy settings for douban project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'douban'

SPIDER_MODULES = ['douban.spiders']
NEWSPIDER_MODULE = 'douban.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
# 修改请求头
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) ' \
             'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36'

# Obey robots.txt rules
# 是否遵守规则
ROBOTSTXT_OBEY = True

# Configure maximum concurrent requests performed by Scrapy (default: 16)
# 并发请求数量
CONCURRENT_REQUESTS = 2

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
# 下载延迟
DOWNLOAD_DELAY = 5
# The download delay setting will honor only one of:
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
# CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
# 允许保存cookie
COOKIES_ENABLED = True

MONGODB_SERVER = '47.98.172.171'
MONGODB_PORT = 27017
MONGODB_DB = 'douban'
MONGODB_COLLECTION = 'movie'


# Disable Telnet Console (enabled by default)
# TELNETCONSOLE_ENABLED = False

# Override the default request headers:
# DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
# }

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
# SPIDER_MIDDLEWARES = {
#    'douban.middlewares.DoubanSpiderMiddleware': 543,
# }

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# 下载中间件
DOWNLOADER_MIDDLEWARES = {
   'douban.middlewares.DoubanDownloaderMiddleware': 543,
}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
# EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
# }

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
# 让PIPELINES生效， 后面的数字表示执行的级别，越小越先执行
ITEM_PIPELINES = {
   'douban.pipelines.DoubanPipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
# AUTOTHROTTLE_ENABLED = True
# The initial download delay
# AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
# AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
# AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
# 缓存爬取过得页面，有助于提升爬虫性能
HTTPCACHE_ENABLED = True
HTTPCACHE_EXPIRATION_SECS = 0
HTTPCACHE_DIR = 'httpcache'
HTTPCACHE_IGNORE_HTTP_CODES = []
HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

• movie.py中
  
  • 这个文件是自己创建的爬虫文件在spiders文件下
  • scrapy genspider movie movie.douban.com 是这个命令创建出来的

# -*- coding: utf-8 -*-
import scrapy

from douban.items import MovieItem


class MovieSpider(scrapy.Spider):
    name = 'movie'
    allowed_domains = ['movie.douban.com']
    start_urls = ['https://movie.douban.com/top250']

    def parse(self, response):
        li_list = response.xpath('//*[@id="content"]/div/div[1]/ol/li')
        for li in li_list:
            item = MovieItem()
            # 在后面加上text()表示去标签里面的内容
            item['title'] = li.xpath('div/div[2]/div[1]/a/span[1]/text()').extract_first()
            item['score'] = li.xpath('div/div[2]/div[2]/div/span[2]/text()').extract_first()
            item['motto'] = li.xpath('div/div[2]/div[2]/p[2]/span/text()').extract_first()
            yield item
        # 用样式表拿
        # a_list = response.css('a[href]::text') 取a标签里面的内容
        # 取属性内容, .re可以加正则表达式
        href_list = response.css('a[href]::attr("href")').re('\?start=.*')
        for href in href_list:
            # 将url补全
            url = response.urljoin(href)
            # 将新的url传给request, 并回调parse方法
            yield scrapy.Request(url=url, callback=self.parse)

• items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class MovieItem(scrapy.Item):

    # 设置要提取出的字段
    title = scrapy.Field()
    score = scrapy.Field()
    motto = scrapy.Field()

• middlewares.py
  
  • 这里是设置了代理，防止封ip
  • 并且是在download中间件下修改process_request

class XiciDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        # 设置代理
        # proxyip放了一些代理ip,通过这种方法可以测试出哪个好用
        # with open('E://proxyip.txt', 'r', encoding='utf-8') as f:
        #     ip_list = f.readlines()
        #     while True:
        #         ip = choice(ip_list).strip()
        #         if ip != 'ip:':
        #             break
        # 将可用的代理ip写入下面的列表中，每次执行文件随机选取
        pro_list = [
            'HTTPS://115.54.204.118:35868',
            'HTTP://171.80.96.49:6666',
            'HTTPS://218.75.114.6:63000',
            'HTTPS://14.118.255.61:6666',
            'HTTPS://223.145.230.84:6666',
            'HTTP://175.155.24.51:808',
            'HTTPS://183.143.91.251:40706',
            'HTTPS://125.120.206.24:6666',
            'HTTPS://123.171.43.142:808',
        ]
        request.meta['proxy'] = choice(pro_list)
        # print(ip)

• pipelines.py中
  
  • 持久化数据，需要在setting中将对应注释取消

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymongo

# 注意settings导入位置，不是在文件目录中的位置
from scrapy.conf import settings

class DoubanPipeline(object):

    def __init__(self):
        connection = pymongo.MongoClient(settings['MONGODB_SERVER'], settings['MONGODB_PORT'])
        db = connection[settings['MONGODB_DB']]
        self.connection = db[settings['MONGODB_COLLECTION']]

    def process_item(self, item, spider):
        self.connection.insert_one(dict(item))

        return item

• 用模板来创建蜘蛛
• scrapy genspider movie movie.douban.com –template=crawl
• 用这种方式创建可以指定要搜索的url,并且在打开的网页中自动提取要搜索的url

import scrapy
from scrapy.selector import Selector
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

from douban.items import DoubanItem


class MovieSpider(CrawlSpider):
    name = 'movie'
    allowed_domains = ['movie.douban.com']
    start_urls = ['https://movie.douban.com/top250']
    rules = (
    # 匹配规则
        Rule(LinkExtractor(allow=(r'https://movie.douban.com/top250\?start=\d+.*'))),
        Rule(LinkExtractor(allow=(r'https://movie.douban.com/subject/\d+')), callback='parse_item'),
    )
    # 这里只需要返回item，不需要再返回url
    def parse_item(self, response):
        return item