使用多进程，多线程和协程实现爬虫，使用queue完成数据通信

一直在看python的多进程，多线程和协程，动手试了试写个爬虫练习一下。

多线程版本 

import ssl
import threading
import time
import urllib.request
from queue import Queue
from lxml import etree


class myThread(threading.Thread):
    def __init__(self,url,q):
        super(myThread, self).__init__()
        self.q = q
        self.headers = {

            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'

        }
        self.url = url

    def run(self):
        self.parse_html()

    def send_request(self, url):
        request = urllib.request.Request(url=url, headers=self.headers)
        #我这里需要加上ssl认证
        ssl._create_default_https_context = ssl._create_unverified_context
        response = urllib.request.urlopen(request)

        html_response = response.read().decode('utf-8')

        return html_response

    def parse_html(self):
        resp = self.send_request(self.url)
        html = etree.HTML(resp)
        div_list = html.xpath("//div[@class='index_only']//div[@id='container']/div")
        for div in div_list:
            name = div.xpath(".//img/@alt")[0]
            src = div.xpath(".//img/@src2")[0]
            # print(src)
            # print(name)
            self.q.put(name + '\t' + src)

def main():
    q = Queue()
    base_url = 'http://sc.chinaz.com/tupian/renwutupian_'

    url_list = [base_url + str(num)+'.html' for num in range(2, 100)]
    print(url_list)
    Thread_list = []
    # 创建并启动线程
    for url in url_list:
        p = myThread(url, q)
        p.start()
        Thread_list.append(p)
  # 让主线程等待子线程执行完成
    for i in Thread_list:
        i.join()
    #打印队列里存入的信息
    while not q.empty():
        print(q.get())


if __name__ == "__main__":
    start = time.time()
    main()
    print('总共耗时：%s' % (time.time() - start))

多进程版本

import multiprocessing
import ssl
import time
import urllib.request
from queue import Queue

from lxml import etree


class ZhanzProcess(multiprocessing.Process):

    def __init__(self, url, q):
        super(ZhanzProcess, self).__init__()

        self.q = q
        self.url = url
        self.headers = {

            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'

        }

    def run(self):
        self.parse_html()

    def send_request(self, url):
        request = urllib.request.Request(url=url, headers=self.headers)
        ssl._create_default_https_context = ssl._create_unverified_context
        response = urllib.request.urlopen(request)
        html_response = response.read().decode('utf-8')

        return html_response

    def parse_html(self):
        res = self.send_request(self.url)
        html = etree.HTML(res)
        div_list = html.xpath("//div[@class='index_only']//div[@id='container']/div")
        for div in div_list:
            name = div.xpath(".//img/@alt")[0]
            src = div.xpath(".//img/@src2")[0]
            # print(name + '\t' + src)

            self.q.put(name + '\t' + src)


def main():
    q = Queue()
    base_url = 'http://sc.chinaz.com/tupian/renwutupian_'

    url_list = [base_url + str(num) + '.html' for num in range(2, 100)]
    print(url_list)
    process_list = []
    for url in url_list:
        # print(url)
        p = ZhanzProcess(url, q)
        p.start()
        process_list.append(p)

    for i in process_list:
        i.join()

    while not q.empty():
         print(q.get())
 


if __name__ == "__main__":
    start = time.time()
    main()
    print('总共耗时：%s' % (time.time() - start))

协程版本

import ssl
import time
import urllib.request
from queue import Queue

import gevent as gevent
from lxml import etree

# 打猴子补丁
from gevent import monkey

monkey.patch_all()


class Zzspider(object):

    def __init__(self):
        self.q = Queue()
        self.headers = {

            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'

        }

    def run(self, url):
        self.parse_html(url)

    def send_request(self, url):
        request = urllib.request.Request(url=url, headers=self.headers)
        ssl._create_default_https_context = ssl._create_unverified_context
        response = urllib.request.urlopen(request)
        html_response = response.read().decode('utf-8')

        return html_response

    def parse_html(self, url):
        res = self.send_request(url)
        html = etree.HTML(res)
        div_list = html.xpath("//div[@class='index_only']//div[@id='container']/div")
        for div in div_list:
            name = div.xpath(".//img/@alt")[0]
            src = div.xpath(".//img/@src2")[0]

            self.q.put(name + '\t' + src)

    def main(self):
        base_url = 'http://sc.chinaz.com/tupian/renwutupian_'

        url_list = [base_url + str(num) + '.html' for num in range(2, 100)]
        job_list = [gevent.spawn(self.run, url) for url in url_list]
        # 让线程等待所有任务完成，再继续执行。
        gevent.joinall(job_list)
        while not self.q.empty():
            print(self.q.get())


if __name__ == '__main__':
    start = time.time()
    zz = Zzspider()
    zz.main()
    print('总共耗时：%s' % (time.time() - start))