python3 多线程爬虫的使用demo

import datetime

import io
import sys
import os
import requests
from queue import Queue
from pymongo import MongoClient
import json
import jsonpath
import threading



#采集线程
class ThreadCrawl(threading.Thread):

    def __init__(self,threadName,cityQueue,dataQueue):
        super(ThreadCrawl,self).__init__()
        self.threadName = threadName
        self.cityQueue = cityQueue
        self.dataQueue = dataQueue

        self.headers = {
            'charset':'utf-8',
            'Accept-Encoding':'gzip',
            'referer':'https://servicewechat.com/wx4a68a5b1b2d89fea/32/page-frame.html',
            'content-type':'application/json',
            'User-Agent':'Mozilla/5.0 (Linux; Android 5.1.1; Redmi 3 Build/LMY47V; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 Mobile Safari/537.36 MicroMessenger/7.0.6.1460(0x27000634) Process/appbrand2 NetType/WIFI Language/zh_CN',
            'Content-Length':'415',
            'Host':'sjz.ihotels.cc',
            'Connection':'Keep-Alive',
        }
        self.json = {"brandNameList":["AA连锁酒店"],
                     "serviceNameList":[],
                     "distance":"",
                     "commentLeave":"",
                     "beginDate":"2019-08-26",
                     "endDate":"2019-08-27",
                     #"cityName":"上海",
                     "areaList":[],
                     "keyword":"",
                     "openType":1,
                     "pageIndex":1,
                     "pageSize":50,
                     "sortOpt":"",
                     "sortType":"",
                     #"lat":40.001064,
                     #"lon":116.461409,
                     "memberId":"",
                     "channel":17,
                     "deviceType":"4",
                     "tagVersion":"5.0.0",
                     "deviceName":"Redmi 3",
                     "code":"023NrRa70BYVxF1c7ra70LBGa70NrRan"
                     }
    def run(self):
        print('启动'+self.threadName+'...')
        while not CRAWL_EXIT:
            try:
                # 取出一个数字，，先进先出
                # 可选参数block，默认是true
                # 1.如果队列为空，block为true不会结束 就会进入阻塞状态，直到队列有新的数据
                # 2.如果队列，block为false的话 就会弹出一个queue.empty()异常
                city = self.cityQueue.get(False)
                print(city+'-----------')
                self.json['cityName'] = city
                listUrl = "https://sjz.ihotels.cc//ethank-sjz-web/rest/hotelResource/v2.1/queryHotelList"
                # print(url)
                content = requests.post(listUrl, headers=self.headers,json=self.json)
                if 'hotelName' in content.text:
                    self.dataQueue.put(content.text)
            except:
                pass
        print("结束 " + self.threadName)

CRAWL_EXIT = False
PARSE_EXIT = False


class ThreadParse(threading.Thread):
    def __init__(self,threadName,dataQueue,lock):
        super(ThreadParse,self).__init__()
        self.threadName = threadName
        self.dataQueue = dataQueue
        self.lock = lock
        self.strings  = []
    def run(self):
        print('启动' + self.threadName + '...')
        while not CRAWL_EXIT:
            try:
                text = self.dataQueue.get(False)
                self.parse(text)
            except:
                pass

        print(self.strings)
        print("结束"+self.threadName)



    def parse(self,text):
        setup_io()
        textjson = json.loads(text,encoding='utf-8')
        jsonList = jsonpath.jsonpath(textjson,'$..hotelList')[0]
        for list in jsonList:
            poi_id = jsonpath.jsonpath(list,'$..hotelId')[0]
            poi_name = jsonpath.jsonpath(list, '$..hotelName')[0]
            city = jsonpath.jsonpath(list,'$..areaName')[0]
            address = jsonpath.jsonpath(list,'$..areaName')[0]
            score = jsonpath.jsonpath(list,'$..score')[0]
            crawl_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            source = "尚美荟"
            keyword = 'AA连锁'
            strings = {'task_id':os.environ.get('CRAWLAB_TASK_ID'),
                       'poi_id':poi_id,
                       'poi_name':poi_name,
                       'city':city,
                       'address':address,
                       'score':score,
                       'crawl_time':crawl_time,
                       'source':source,
                       'keyword':keyword}
            print(strings)
            self.strings.append(strings)

#编码转换错误解决方式
def setup_io():
    sys.stdout = sys.__stdout__ = io.TextIOWrapper(sys.stdout.detach(), encoding='utf-8', line_buffering=True)
    sys.stderr = sys.__stderr__ = io.TextIOWrapper(sys.stderr.detach(), encoding='utf-8', line_buffering=True)
#程序主函数
def mian(citylist):
    setup_io()#编码转换错误解决方式
    cityjson = json.loads(citylist, encoding='utf-8')#把列表转为json字符串
    citys = jsonpath.jsonpath(cityjson, '$..city')#获取所有的city
    print(citys)
    #创建城市队列
    cityQueue = Queue()
    for i in citys:
        cityQueue.put(i)
    #创建数据队列
    dataQueue = Queue()
    #创建锁
    lock = threading.local
    #创建采集线程
    crawlList = ['采集线程1号','采集线程2号','采集线程3号']
    threadcrawl = [];
    for threadName in crawlList:
        thread = ThreadCrawl(threadName, cityQueue,dataQueue)
        thread.start()
        threadcrawl.append(thread)

    #创建解析线程
    parseList = ['解析线程1号','解析线程2号','解析线程3号']
    threadparse = [];
    for threadName in parseList:
        thread = ThreadParse(threadName,dataQueue,lock)
        thread.start()
        threadparse.append(thread)
    while not cityQueue.empty():
        pass
    # 如果cityQueue为空，采集线程退出循环
    global CRAWL_EXIT
    CRAWL_EXIT = True
    print("cityQueue为空")
    # 让抓取主线程进入阻塞状态，等待子线程执行完毕再退出
    for thread in threadcrawl:
        thread.join()
        print("1")
    # 如果dataQueue为空，采集线程退出循环
    while not dataQueue.empty():
        pass
    global PARSE_EXIT
    PARSE_EXIT = True
    #让数据主线程进入阻塞状态，等待子线程执行完毕再退出
    for thread in threadparse:
        thread.join()
        print("2")
    print("谢谢使用！")

#程序入口
if __name__ == '__main__':
    cityjson=[ {
            "city": "成都",
            "initial": "c"
            }, {
            "city": "上海",
            "initial": "s"
            }, {
            "city": "中卫",
            "initial": "Z"
            },{
                "city": "重庆",
                "initial": "C"
            }, {
                "city": "承德",
                "initial": "C"
            }, {
                "city": "沧州",
                "initial": "C"
            }, {
                "city": "长治",
                "initial": "C"
            }, {
                "city": "赤峰",
                "initial": "C"
            }, {
                "city": "朝阳",
                "initial": "C"
            }, {
                "city": "长春",
                "initial": "C"
            }, {
                "city": "常州",
                "initial": "C"
            }, {
                "city": "滁州",
                "initial": "C"
            }, {
                "city": "巢湖",
                "initial": "C"
            }, {
                "city": "池州",
                "initial": "C"
            }, {
                "city": "长沙",
                "initial": "C"
            }
    ]
    json_str = json.dumps(cityjson, ensure_ascii=False) #[{"city": "阿拉善盟", "initial": "A"}, {"city": "鞍山", "initial": "A"}, {"city": "安庆", "initial": "A"}, {"city": "中卫", "initial": "Z"}]
    mian(json_str)