python用协程池异步爬取音乐的json数据

# -*- coding: utf-8 -*-

# @Author  : Acm

import gevent.monkey
gevent.monkey.patch_all()
from gevent.pool import Pool
from Queue import Queue
import json
import jsonpath
import requests
import time
import random


class HaoLingSheng(object):

    user_agent_list = [
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
        "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
        "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
        "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
        "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
        "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
        "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10",
        "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/533.17.8 (KHTML, like Gecko) Version/5.0.1 Safari/533.17.8",
        "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
        "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.1.17) Gecko/20110123 (like Firefox/3.x) SeaMonkey/2.0.12",
        "Mozilla/5.0 (Windows NT 5.2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1",
        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.8 (KHTML, like Gecko) Chrome/4.0.302.2 Safari/532.8",
        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.464.0 Safari/534.3",
        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.15 Safari/534.13",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.54 Safari/535.2",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",
        "Mozilla/5.0 (Macintosh; U; Mac OS X Mach-O; en-US; rv:2.0a) Gecko/20040614 Firefox/3.0.0 ",
        "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.0.3) Gecko/2008092414 Firefox/3.0.3",
        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-US; rv:1.9.1) Gecko/20090624 Firefox/3.5",
        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.14) Gecko/20110218 AlexaToolbar/alxf-2.0 Firefox/3.6.14",
        "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.2.15) Gecko/20110303 Firefox/3.6.15",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"
    ]

    # 高匿ip代理
    # proxies = "http://61.135.217.7:80"

    def __init__(self):
        self.base_url = "http://m.haolingsheng.com/jsonp/sp.asp?"
        self.mp3_url = "http://m.haolingsheng.com/lingsheng/{}.htm"
        self.headers = {"User-Agent": HaoLingSheng.user_agent_list[random.randint(0, len(HaoLingSheng.user_agent_list))]}
        # 创建一个空列表放置信息
        self.my_list = []
        # 存解析出来的字典类型数据
        self.my_dict_list = []

        self.queue = Queue()
        self.pool = Pool(5)
        self.is_running = True
        self.total_requests_num = 0
        self.total_response_num = 0

    def get_url_params(self):
        # 遍历访问页，参数形式
        for i in range(1, 400):
            params = {
                "callback": "callback",
                # "type": "zuixin",
                # "type":"tuijian",
                "type":"paihang",
                "page": i,
            }
            # 将参数存入队列中，等待调取
            self.queue.put(params)
            self.total_requests_num += 1

    def parse_url(self, params):
        # 解析出数据并转为字典格式
        response = requests.get(url=self.base_url, headers=self.headers, params=params) # proxies=HaoLingSheng.proxies
        ret_str = response.content.decode('unicode_escape')
        ret_str = ret_str.lstrip("callback(").rstrip(")")
        goal_dict = eval(ret_str)
        return goal_dict

    def save_content_list(self, goal_dict):
        # 拿到每一页的数据(20个)
        result = jsonpath.jsonpath(goal_dict, '$.result')[0]
        # 拿出每一个数据加入到自己创建的列表中
        for j in range(len(result)):
            # 将链接加入数据中
            result[j]["url"] = self.mp3_url.format(result[j]["shorturl"])
            # 删除无用链接
            del result[j]["shorturl"]
            self.my_list.append(result[j])

    def save_file(self):
        # 将有数据的列表，写入文件，json格式
        with open("5_paihang.json", "w") as f:
            json.dump(self.my_list, f, ensure_ascii=False, indent=2)

    def execute_requests_item_save(self):
        # 在Queue队列中拿参
        params = self.queue.get()
        # 解析数据
        goal_dict = self.parse_url(params)
        # 保存数据
        self.save_content_list(goal_dict)
        self.total_response_num += 1
        print self.total_response_num

    def _callback(self, temp):
        if self.is_running:
            self.pool.apply_async(self.execute_requests_item_save, callback=self._callback)

    def run(self):
        self.get_url_params()

        for i in range(5):
            self.pool.apply_async(self.execute_requests_item_save, callback=self._callback)

        while True:
            time.sleep(0.0001)
            if self.total_response_num >= self.total_requests_num:
                self.is_running = False
                self.save_file()
                break


if __name__ == '__main__':
    start = time.time()
    haoling = HaoLingSheng()
    haoling.run()
    end = time.time()

    print end-start