multiprocessing 爬取京东内衣数据 json数据 ， 保存为 excel

multiprocessing 爬取京东内衣数据 json数据 ， 保存为 excel

import requests
import json
from openpyxl import Workbook,load_workbook
from multiprocessing import Pool


wb = Workbook()
sheet = wb.active
sheet.title = "neiyii"
wb.save("neiyii.xlsx")

headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36"}
def ges(url):
    try:
        response = requests.get(url,headers=headers).text
        html = response[20:-2] # 切片很关键

        json_data = json.loads(html)
        jso = json_data["comments"]
        for i in jso:

            productSize = i["productSize"]
            productColor = i["productColor"]
            reference_time = i["referenceTime"]
            reference_name = i["referenceName"]
            print(reference_time)
            sheet.append([productSize,productColor,reference_name,reference_time])
            wb.save("neiyii.xlsx")
    except Exception as e:
        print(e)


if __name__ == '__main__':
    pool = Pool(6)

    urls= ["https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=100002038213&score=0&sortType=5&page={}&pageSize=10&isShadowSku=0&rid=0&fold=1".format(i) for i in range(100)]
    pool.map(ges,urls)