python京东爬虫

#json数据格式问题，可能是请求过于频繁，网站没有返回正常的数据
#每天早上起床时可以运行，到下午后就一直报错。
import requests
import json
import csv       
for page_num in range(49):
    new_url = f'https://club.jd.com/comment/productPageComments.action?productId=10023108638660&score=0&sortType=5&page={page_num}&pageSize=10&isShadowSku=0&rid=0&fold=1'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'
}
#
for page_num in range(49):
    new_url = f'https://club.jd.com/comment/productPageComments.action?productId=10023108638660&score=0&sortType=5&page={page_num}&pageSize=10&isShadowSku=0&rid=0&fold=1'
    response = requests.get(new_url, headers=headers)
    js_data=json.loads(response.text)
    #js_data=response.json()
    comment_list=js_data['comments']
    for comment in comment_list:
        goods_id = comment.get('id')
        nickname = comment.get('nickname')
        score = comment.get('score')
        productSize = comment.get('productSize')
        productColor = comment.get('productColor')
        creationTime = comment.get('creationTime')
        content = comment.get('content')
        content = ' '.join(content.split('\n'))
        with open('gpc_pcaaa123123.csv', 'a+', newline='', encoding='gb18030') as f:
            writer = csv.writer(f)
            writer.writerow((goods_id, nickname, score, productSize, productColor, creationTime,content))
    f.close()

import re
import csv
import os
import requests
import html
headers = {
    'cookie': 'shshshfp=22dd633052035d21be92463ffa35684d; shshshfpa=ab283f84-c40f-9710-db89-84a8d3366a81-1586333030; __jda=122270672.1586333031101106032014.1586333031.1586333031.1586333031.1; __jdv=122270672|direct|-|none|-|1586333031103; __jdc=122270672; shshshfpb=bUe7tI9%2FOOaJKd7vP0EtSOg%3D%3D; __jdu=1586333031101106032014; areaId=22; ipLoc-djd=22-1977-1980-0; 3AB9D23F7A4B3C9B=7XEQD4BFTGEH44EK7LN7HLFCHJW6W2NS5VJOQOCHABZVI7LXJJIW3K2IX5MTPZ4TBERBLY6TRQR5CA3S3IYVLQ2JGI; jwotest_product=99; shshshsID=a7457cee6a4a9fa285fe2cff44c6bd17_4_1586333142454; __jdb=122270672.4.1586333031101106032014|1.1586333031; JSESSIONID=8C21549A613B83F0CB86EF1F38FD63D3.s1',
    'sec-fetch-dest': 'document',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-site': 'none',
    'sec-fetch-user': '?1',
    'upgrade-insecure-requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.162 Safari/537.36'
}
with open('aergs.csv', 'a+', newline='', encoding='gb18030') as f:
    writer = csv.writer(f)
    writer.writerow(('留言时间', '评分', '回复数', '点赞数', '图片数', '评论内容'))
    for page in range(100):
        print('当前正在下载第%d页评论' % (page + 1))
        url = 'https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=100000499657&score=0&sortType=5&page=%d&pageSize=10&isShadowSku=0&fold=1' % page
        text = requests.get(url, headers=headers).text
        comment_list = re.findall(r'guid":".*?"content":"(.*?)".*?"creationTime":"(.*?)",".*?"replyCount":(\d+),"score":(\d+).*?usefulVoteCount":(\d+).*?imageCount":(\d+).*?images":',text)
        for result in comment_list:
            # 根据正则表达式结果匹配数据
            content = html.unescape(result[0]).replace('\n', ' ')
            comment_time = result[1]
            reply_count = result[2]
            score = result[3]
            vote_count = result[4]
            image_count = result[5]
            # 写入数据
            writer.writerow((comment_time, score, reply_count, vote_count, image_count, content))