协程greenlet+协程下载


'''
  协程greenlet的不同人物之间的切换:

'''
from time import sleep

from greenlet import greenlet


def eat():
    for i in range(5):
        print('正在吃第{}个馒头'.format(i + 1))
        # sleep(0.6)
        g2.switch()  # 任务的切换
        print('---------------')


def listen():
    for i in range(5):
        print('正在听第{}首歌'.format(i + 1))
        g1.switch()


if __name__ == '__main__':
    g1 = greenlet(eat)
    g2 = greenlet(listen)
    #
    g1.switch()  # 要完成谁的任务就要切换到哪个greenlet上

'''
使用协程下载
'''
import os
import random
import ssl
import time

import gevent
import requests
import urllib.request
from gevent import monkey

monkey.patch_all()

agents = [
    'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
    'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1'
    'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Mobile Safari/537.36'
]


# 网络下载
def download(url):
    context = ssl._create_unverified_context()
    # response = requests.get(url, headers={'User-Agent': random.choice(agents)}, verify=False)
    # return response.text
    request = urllib.request.Request(url,headers={'User-Agent': random.choice(agents)})
    response = urllib.request.urlopen(request,context=context)
    return response.read().decode()


# 本地存储
def download_page(url):
    filename = os.path.split(url)[1] + '.html'
    with open(filename, 'w',encoding='utf-8') as ws:
        content = download(url)
        ws.write(content)


if __name__ == '__main__':
    os.mkdir('lianjia')
    os.chdir('lianjia')
    url = 'https://bj.lianjia.com/ershoufang/pg'
    # https://bj.lianjia.com/ershoufang/pg3   pg3.html
    start = time.time()
    glist = []
    for i in range(1, 11):
        g = gevent.spawn(download_page, url + str(i))
        glist.append(g)

    for g in glist:
        g.join()

    end = time.time()
    print('用时:{}'.format(end - start))

 

发布了259 篇原创文章 · 获赞 6 · 访问量 3938
App 阅读领勋章
微信扫码 下载APP
阅读全文

没有更多推荐了,返回首页

©️2019 CSDN 皮肤主题: 数字20 设计师: CSDN官方博客

分享到微信朋友圈

×

扫一扫,手机浏览