python 多线程采集amac

本文用到的知识点

•  通过requests发送POST请求，参数以request payload方式进行传输
• 时间戳转为格式化日期
• 通过openpyxl 将数据保存为excel
• 使用multiprocessing.dummy.Pool 多线程采集

目标地址 

import requests
import json
import threading
from multiprocessing.dummy import Pool
from openpyxl import Workbook
import time


class MyProcess(object):
    
    def __init__(self):
        self.headers = {
            "Content-Type": "application/json",
            "Host": "gs.amac.org.cn",
            "Referer": "http://gs.amac.org.cn/amac-infodisc/res/pof/manager/managerlistFormap.html?province=%E5%B9%BF%E4%B8%9C&city=%E6%B7%B1%E5%9C%B3%E5%B8%82&tdsourcetag=s_pctim_aiomsg",
            "Origin": "http://gs.amac.org.cn",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36",
            "X-Requested-With": "XMLHttpRequest"
        }
        self.workbook = Workbook()
        self.sheet = self.workbook.create_sheet('amac', index=0)
        self.lock = threading.Lock()

    def open_spider(self):
        title = ("私募基金管理人名称", "法定代表人/执行事务合伙人(委派代表)姓名", "办公地址", "注册地址", "注册地",
                 "登记编号", "成立时间", "登记时间")
        self.sheet.append(title)

    def get_response(self, url):

        data = {
            "registerCity": "深圳市",
            "registerProvince": "广东"
        }
        response = requests.post(url, headers=self.headers, data=json.dumps(data))
        return response.text

    def parse(self, url):
        time.sleep(1)
        results = json.loads(self.get_response(url))
        for content in results.get('content'):
            managerName = content.get("managerName")
            artificialPersonName = content.get("artificialPersonName")
            officeAddress = content.get("officeAddress")
            registerAddress = content.get("registerAddress")
            regAdrAgg = content.get("regAdrAgg")
            registerNo = content.get("registerNo")
            establishDate = content.get("establishDate")
            establishDate = self.handle_date(establishDate)
            registerDate = content.get("registerDate")
            registerDate = self.handle_date(registerDate)
            value = (managerName, artificialPersonName, officeAddress, registerAddress, regAdrAgg,
                     registerNo, establishDate, registerDate)
            self.save(value)

    @staticmethod
    def handle_date(date):
        d = time.localtime(int(str(date)[:10]))
        t = time.strftime('%Y-%m-%d', d)
        return t

    def save(self, value):
        self.lock.acquire()
        self.sheet.append(value)
        self.lock.release()

    def close_spider(self):
        self.workbook.save('amac.xlsx')

    def run(self):
        self.open_spider()
        pool = Pool(processes=3)
        uri = "http://gs.amac.org.cn/amac-infodisc/api/pof/manager?rand=0.2130710083741889&page={}&size=50"
        for i in range(10):
            url = uri.format(i)
            pool.apply_async(self.parse, args=(url,))
        pool.close()
        pool.join()
        self.close_spider()


if __name__ == '__main__':
    mp = MyProcess()
    mp.run()