我使用python的进程池技术下载企业工商数据，速度1000万条/天，超快！

这次主要是采用python的多线程、进程池技术，快速下载企业工商数据的方法

# 每个线程负责爬取10万个url，那么开启100个线程，速度分分秒秒把企业工商数据全部跑下来，您电脑cpu和内存好的话，一天可以跑1000多万。这个python性能不错，感觉很爽！

核心代码：
# pool = threadpool.ThreadPool(100个线程)
# mrequests = threadpool.makeRequests(Spider().get_gongshang, urls)
# [pool.putRequest(req) for req in mrequests]
# pool.wait()

跑下来的数据结构如下：

{
    "url": "http://www.xxx.com/firm_0a6ca0be1266656b93d03390abf4e40b.html",
    "基本信息": {
        "人员规模：": "-",
        "企业地址：": "温州市鹿城区蒲鞋市街道车站大道733号",
        "公司名": "温州均瑶集团有限公司",
        "公司类型：": "有限责任公司(自然人投资或控股)",
        "地址": "温州市鹿城区蒲鞋市街道车站大道733号",
        "官网": "http://www.juneyao.com",
        "成立日期：": "1995-08-23",
        "所属地区": "浙江省",
        "所属行业": "住宿和餐饮业",
        "曾用名：": "均瑶集团有限公司",
        "核准日期：": "2016-12-30",
        "法定代表人信息": {
            "pl_url": "http://www.xxx.com/pl_p808d5b5f8c083541230a8951a8866aa.html",
            "他关联公司数": "39",
            "姓名": "王均豪"
        },
        "注册号：": "330305000013113",
        "注册资本：": "8850万人民币",
        "登记机关：": "温州市市场监督管理局",
        "登记状态": "存续",
        "纳税人识别号：": "91330300145067169Y",
        "组织机构代码：": "14506716-9",
        "经营状态：": "存续",
        "经营范围：": "住宿服务（限下设分支机构经营）。  对实业项目的投资。",
        "统一社会信用代码：": "91330300145067169Y",
        "联系方式": "0577-88098816",
        "英文名：": "-",
        "营业期限：": "1995-08-23 至 2045-08-22",
        "邮箱": "暂无"
    },
    "主要成员信息": [
        {
            "pl_url": "http://www.xxx.com/pl_p7663e2e7b8bcd23cfa9e1d85eac902a.html",
            "他关联公司数": "15",
            "姓名": "裴学龙",
            "职务": "监事"
        },
        {
            "pl_url": "http://www.xxx.com/pl_p75b867211714a967474c4ac4706e918.html",
            "他关联公司数": "11",
            "姓名": "王瀚",
            "职务": "董事"
        },
        {
            "pl_url": "http://www.xxx.com/pl_pa2912161d6acaed1baea9e26e08e062.html",
            "他关联公司数": "12",
            "姓名": "钱克流",
            "职务": "董事"
        },
        {
            "pl_url": "http://www.xxx.com/pl_pad4f49d512d91e830c1fcf76235e986.html",
            "他关联公司数": "5",
            "姓名": "陈体理",
            "职务": "监事"
        },
        {
            "pl_url": "http://www.xxx.com/pl_p808d5b5f8c083541230a8951a8866aa.html",
            "他关联公司数": "39",
            "姓名": "王均豪",
            "职务": "-"
        },
        {
            "pl_url": "http://www.xxx.com/pl_pc97b89c41134307548eb11de6430258.html",
            "他关联公司数": "23",
            "姓名": "蒋海龙",
            "职务": "监事"
        },
        {
            "pl_url": "http://www.xxx.com/pl_pd1515f7945aecbfe2bdafdf098bc819.html",
            "他关联公司数": "32",
            "姓名": "王均金",
            "职务": "-"
        },
        {
            "pl_url": "http://www.xxx.com/pl_p2a05f8d97284529e0ad9f176a603db8.html",
            "他关联公司数": "14",
            "姓名": "林乃机",
            "职务": "董事"
        }
    ],
    "公司名": "温州均瑶集团有限公司",
    "其他链接": {
        "企业年报": {
            "count": "4",
            "url": "http://www.xxx.com/creport_0a6ca0be1266656b93d03390abf4e40b"
        },
        "对外投资": {
            "count": "12",
            "url": "http://www.xxx.com/cinvestment_0a6ca0be1266656b93d03390abf4e40b"
        },
        "法律诉讼": {
            "count": "0",
            "url": "http://www.xxx.com/clawsuit_0a6ca0be1266656b93d03390abf4e40b"
        },
        "知识产权": {
            "count": "106",
            "url": "http://www.xxx.com/cassets_0a6ca0be1266656b93d03390abf4e40b"
        },
        "经营状况": {
            "count": "7",
            "url": "http://www.xxx.com/crun_0a6ca0be1266656b93d03390abf4e40b"
        }
    },
    "分支机构信息": [
        {
            "company": "温州均瑶集团有限公司均瑶宾馆",
            "url": "http://www.xxx.com/firm_2fd416c499d26a28b934e2c7af7b4c47.html"
        },
        {
            "company": "温州均瑶集团有限公司均瑶宾馆",
            "url": "http://www.xxx.com/firm_2fd416c499d26a28b934e2c7af7b4c47.html"
        },
        {
            "company": "温州均瑶集团有限公司兰州分公司",
            "url": "http://www.xxx.com/firm_cfeba3d00d1cfc9483791d1db6ba9b2b.html"
        },
        {
            "company": "温州均瑶集团有限公司武汉分公司",
            "url": "http://www.xxx.com/firm_7254f781af1c076cfa181050349c75c7.html"
        },
        {
            "company": "温州均瑶集团有限公司杭州分公司",
            "url": "http://www.xxx.com/firm_73a2dc6bc97566f04bb31224ea99b9a2.html"
        },
        {
            "company": "温州均瑶集团有限公司西安分公司",
            "url": "http://www.xxx.com/firm_9112b5ad54de9b2a192351e97a17bfda.html"
        },
        {
            "company": "温州均瑶集团有限公司无锡分公司",
            "url": "http://www.xxx.com/firm_b5522b2f6f333f4fe080184344d68181.html"
        }
    ],
    "变更信息": [
        {
            "变更项目": "历史法定代表人",
            "姓名": "王均金"
        },
        {
            "变更前": "王均豪*,钱克流,王均金,许彪,陈体理,蒋海龙,裴学龙",
            "变更后": "王均豪*,钱克流,王均金,陈体理,蒋海龙,裴学龙,王瀚,林乃机",
            "变更日期": "2016-12-30",
            "变更项目": "高级管理人员备案  带有*标记的为法定代表人",
            "序号": "1"
        },
        {
            "变更前": "原章程",
            "变更后": "2016年6月23日章程修订本",
            "变更日期": "2016-12-30",
            "变更项目": "章程修正案备案",
            "序号": "1"
        },
        {
            "变更前": "注册号:330305000013113组织机构代码证:145067169",
            "变更后": "统一社会信用代码:91330300145067169Y",
            "变更日期": "2016-04-08",
            "变更项目": "换发统一社会信用代码执照",
            "序号": "1"
        },
        {
            "变更前": "一般经营项目:工艺美术品、其他日用品、装饰材料、食品机械设备的销售 机票代理服务。",
            "变更后": "一般经营项目:对实业项目的投资。",
            "变更日期": "2013-08-23",
            "变更项目": "一般经营项目变更",
            "序号": "1"
        },
        {
            "变更前": "许可经营项目:",
            "变更后": "许可经营项目:住宿服务(限下设分支机构经营)。",
            "变更日期": "2012-06-18",
            "变更项目": "许可经营项目变更",
            "序号": "1"
        },
        {
            "变更前": "住所:****号\n                                        邮政编码:325011\n                                        电话:********\n                                        住所:****",
            "变更后": "住所:****号\n                                        邮政编码:325011\n                                        电话:********\n                                        住所:****",
            "变更日期": "2008-11-03",
            "变更项目": "住所(营业场所、地址)变更",
            "序号": "1"
        },
        {
            "变更前": "王均金",
            "变更后": "王均豪*",
            "变更日期": "2005-04-14",
            "变更项目": "法定代表人(负责人、董事长、首席代表)变更  带有*标记的为法定代表人",
            "序号": "1"
        },
        {
            "变更前": "姓名:倪细伦\n                                        证件名称:\n                                        证件号码:********\n                                        性别:男\n                                        职务:董事",
            "变更后": "姓名:裴学龙\n                                        证件名称:\n                                        证件号码:********\n                                        性别:男\n                                        职务:监事",
            "变更日期": "2004-11-22",
            "变更项目": "法定代表人(负责人、董事长、首席代表)变更  带有*标记的为法定代表人",
            "序号": "2"
        },
        {
            "变更前": "均瑶集团有限公司",
            "变更后": "温州均瑶集团有限公司",
            "变更日期": "2004-03-02",
            "变更项目": "名称变更",
            "序号": "1"
        },
        {
            "变更前": "温州市均瑶集团有限公司",
            "变更后": "均瑶集团有限公司",
            "变更日期": "1998-07-22",
            "变更项目": "名称变更",
            "序号": "2"
        },
        {
            "变更前": "经营范围:食品,非酒精饮料,机票代理,塑料制品、塑料包装,工艺品,航空用品,日用百货,五金家电,化工原料(不含危险品),礼品,装璜材料,家具,庆上用品,针织品,服装,游艺活动(须持许可证方可经营)食品机械,餐饮(须持卫生许可证方可经营)。行业代码:1439",
            "变更后": "经营范围:生产销售食品,非酒精饮料,机票代理,塑料制品、塑料包装,工艺品,航空用品,日用百货,五金家电,化工原料(不含危险品),礼品,装璜材料,家具,庆上用品,针织品,服装,游艺活动(须持许可证方可经营)食品机械,餐饮(须持卫生许可证方可经营)。附设分公司经营范围:餐饮、住宿、KTV、歌舞厅、台球、美容美发、电子游戏 保龄球、健美、健美操、运动保健按摩。行业代码:1439",
            "变更日期": "1999-07-01",
            "变更项目": "经营范围(业务范围)变更",
            "序号": "1"
        },
        {
            "变更前": "经营范围:食品,非酒精饮料,机票代理,塑料制品、塑料包装,工艺品,航空用品。行业代码:1439",
            "变更后": "经营范围:食品,非酒精饮料,机票代理,塑料制品、塑料包装,工艺品,航空用品,日用百货,五金家电,化工原料(不含危险品),礼品,装璜材料,家具,庆上用品,针织品,服装,游艺活动(须持许可证方可经营)食品机械,餐饮(须持卫生许可证方可经营)。行业代码:1439",
            "变更日期": "1997-04-29",
            "变更项目": "经营范围(业务范围)变更",
            "序号": "2"
        },
        {
            "变更前": "2050",
            "变更后": "8850",
            "变更日期": "1998-06-01",
            "变更项目": "注册资本(金)变更",
            "序号": "1"
        }
    ],
    "对外投资信息": [
        {
            "url": "http://www.xxx.com/firm_766b85e5c8b26c022f981bdd528fd4e9.html",
            "投资公司": "均瑶集团出租汽车有限公司",
            "持股比例": "90.0%",
            "股东类型": "企业法人",
            "认缴出资日期": "1998-11-03",
            "认缴出资额（万元）": "900"
        },
        {
            "url": "http://www.xxx.com/firm_756f1439563684c178c87bcdc4a7e84e.html",
            "投资公司": "均瑶集团上海有限公司",
            "持股比例": "60.0%",
            "股东类型": "企业法人",
            "认缴出资日期": "-",
            "认缴出资额（万元）": "120"
        },
        {
            "url": "http://www.xxx.com/firm_f62ffa20abdd72a393801d9b75958bdb.html",
            "投资公司": "均瑶集团航空服务有限公司",
            "持股比例": "10.0%",
            "股东类型": "企业法人",
            "认缴出资日期": "-",
            "认缴出资额（万元）": "100"
        },
        {
            "url": "http://www.xxx.com/firm_498d2af5c593b5ca293e46c0f50d97e7.html",
            "投资公司": "均瑶集团乳业股份有限公司",
            "持股比例": "-",
            "股东类型": "企业法人",
            "认缴出资日期": "2010-07-22",
            "认缴出资额（万元）": "-"
        },
        {
            "url": "http://www.xxx.com/firm_fe5ea392f0f4477b90c3eb70be7cb7fc.html",
            "投资公司": "上海均瑶航空投资有限公司",
            "持股比例": "50.0%",
            "股东类型": "企业法人",
            "认缴出资日期": "2002年8月12日",
            "认缴出资额（万元）": "5400"
        },
        {
            "url": "http://www.xxx.com/firm_b52f27fda2943d818c6d2ffbde8b9640.html",
            "投资公司": "温州均瑶商务服务有限公司",
            "持股比例": "20.0%",
            "股东类型": "法人股东",
            "认缴出资日期": "1997-05-23",
            "认缴出资额（万元）": "10 万元"
        },
        {
            "url": "http://www.xxx.com/firm_75d3cea813c7487dc2dc476934f8045d.html",
            "投资公司": "温州瑶住酒店管理有限公司",
            "持股比例": "40.0%",
            "股东类型": "企业法人",
            "认缴出资日期": "-",
            "认缴出资额（万元）": "20"
        },
        {
            "url": "http://www.xxx.com/firm_52c347f6739232f68430a61f349d0e0f.html",
            "投资公司": "温州均瑶物业服务有限公司",
            "持股比例": "100%",
            "股东类型": "企业法人",
            "认缴出资日期": "-",
            "认缴出资额（万元）": "-"
        },
        {
            "url": "http://www.xxx.com/firm_1d75185b284c1bde2663c75e8f7ec65b.html",
            "投资公司": "温州市天龙包机实业有限公司",
            "持股比例": "51.00%",
            "股东类型": "企业法人",
            "认缴出资日期": "1991-12-29",
            "认缴出资额（万元）": "306"
        },
        {
            "url": "http://www.xxx.com/firm_ed495f1f911e0602a215c686b2f44c7a.html",
            "投资公司": "宜昌均瑞房地产开发有限公司",
            "持股比例": "77.00%",
            "股东类型": "企业法人",
            "认缴出资日期": "2012/10/16 0:00:00",
            "认缴出资额（万元）": "7700"
        },
        {
            "url": "http://www.xxx.com/firm_8a82d0cdc7d8b401c3b8c80811dfcb77.html",
            "投资公司": "宜昌均瑶贸易有限公司",
            "持股比例": "100%",
            "股东类型": "企业法人",
            "认缴出资日期": "2025年12月31日",
            "认缴出资额（万元）": "46000"
        },
        {
            "url": "http://www.xxx.com/firm_032731387625750f57f84067e3cfea41.html",
            "投资公司": "深圳市均瑶奶品有限公司",
            "持股比例": "70.0%",
            "股东类型": "法人股东",
            "认缴出资日期": "-",
            "认缴出资额（万元）": "140万元"
        }
    ],
    "股东信息": [
        {
            "pl_url": "http://www.xxx.com/pl_p75b867211714a967474c4ac4706e918.html",
            "他关联公司数": "11",
            "持股比例": "38.500%",
            "股东(发起人)": "王瀚",
            "股东类型": "自然人股东",
            "认缴出资日期": "1997-04-14",
            "认缴出资额（万元）": "3407.25"
        },
        {
            "pl_url": "http://www.xxx.com/pl_pd1515f7945aecbfe2bdafdf098bc819.html",
            "他关联公司数": "32",
            "持股比例": "35.00%",
            "股东(发起人)": "王均金",
            "股东类型": "自然人股东",
            "认缴出资日期": "1998-03-31",
            "认缴出资额（万元）": "3097.5"
        },
        {
            "pl_url": "http://www.xxx.com/pl_p808d5b5f8c083541230a8951a8866aa.html",
            "他关联公司数": "39",
            "持股比例": "25.00%",
            "股东(发起人)": "王均豪",
            "股东类型": "自然人股东",
            "认缴出资日期": "1998-03-31",
            "认缴出资额（万元）": "2212.5"
        },
        {
            "pl_url": "http://www.xxx.com/pl_p68ed1b7c5c7f645c5ac7a1e5dec44dd.html",
            "他关联公司数": "3",
            "持股比例": "0.500%",
            "股东(发起人)": "王超",
            "股东类型": "自然人股东",
            "认缴出资日期": "1997-04-14",
            "认缴出资额（万元）": "44.25"
        },
        {
            "pl_url": "http://www.xxx.com/pl_pe3c8177ddbd2ed2f2080fddb9d9c806.html",
            "他关联公司数": "3",
            "持股比例": "0.500%",
            "股东(发起人)": "王宝弟",
            "股东类型": "自然人股东",
            "认缴出资日期": "1997-04-14",
            "认缴出资额（万元）": "44.25"
        },
        {
            "pl_url": "http://www.xxx.com/pl_p6269be8f67e18ab3bc1403e941d86ba.html",
            "他关联公司数": "5",
            "持股比例": "0.500%",
            "股东(发起人)": "王滢滢",
            "股东类型": "自然人股东",
            "认缴出资日期": "1997-04-14",
            "认缴出资额（万元）": "44.25"
        }
    ]
}

一直从事python多进程、线程池性能、大数据深度技术研究，欢迎更多对大数据技术感兴趣朋友交流：qq2779571288