多线程抓取福布斯富豪榜_福布斯富豪榜api-CSDN博客

本文链接：https://blog.csdn.net/muziiiii/article/details/138430764

# 1. 如何提取单个页面的数据
# 2.上线程池，多个页面同时抓取
import time
from concurrent.futures import ThreadPoolExecutor

import requests
def downloadOnePage(url, rank):
    resp = requests.get(url)
    # print(resp.status_code)

    # 获取20个富豪的json 数据
    richPeopleJson = resp.json()
    # 将json 数据，近一步分割成成一个数组(20个富豪)
    richPeopleListJson20 = richPeopleJson['rows']
    # print(richList20)

    # 存放，整理好的数据，富豪排行
    richPeopleList20 = []

    for i in range(20):
        # 获取每一个富豪的名字，和财富值
        name = richPeopleListJson20[i]['hs_Character'][0]['hs_Character_Fullname_Cn']
        rich = richPeopleListJson20[i]['hs_Rank_Rich_Wealth']
        # print(name)
        # print(rich)
        richPeople = {"name": name, "rich": rich, "rank": rank}
        richPeopleList20.append(richPeople)
        rank += 1

    for r in richPeopleList20:
        print(r)


if __name__ == '__main__':

    # 测试运行时间
    start_time = time.time()

    with ThreadPoolExecutor(32) as t:
        for i in range(0, 62):
            # 第二个参数为富豪的排行
            t.submit(downloadOnePage,
                     f"https://www.hurun.net/zh-CN/Rank/HsRankDetailsList?num=16BKYYA3&search=&offset={i * 20}&limit=20",   (i + 1) * 20 - 19)

    end_time = time.time()
    # 计算程序运行时间
    duration = end_time - start_time
    print(duration)