如何获取json的payload内容，并爬取网易云的python课程信息

爬虫案例:网易云Python课程

昨天写了一个案例，用json数据去爬取网易云课堂的课程内容，使用了payload和headers，但是网页内容太多，payload很难找到。
找一下网页内容，按F12进入开发者模式：

给出代码：

FileName = "制表"
''''''
import requests
import xlsxwriter
def get_json(index):
    data = '''
    爬取课程的json数据
    :param index:当前索引，从0开始
    :return: Json数据
    '''
    url = 'https://study.163.com/p/search/studycourse.json'
    payload = {
        "activityID": 0,
        "keyword": "python",
        "orderType": 5,
        "pageIndex": index + 1,
        "pageSize": 50,
        "priceType": 0,
        "relativeOffset": 0,
        "searchTimeType": -1,
    }

    headers = {
        "accept": "application/json",
        "host": "study.163.com",
        "contentType": "application/json",
        "origin": "https://study.163.com",
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36"
    }
    try:
        response = requests.post(url, json=payload, headers=headers)  # 发送post请求
        # 获取JSON数据
        content_json = response.json()
        # print(content_json)
        if content_json and content_json['code']==0:#判断数据是否存在
            return content_json
        else:
            return None
    except Exception as e:
        print("出错了")
        print(e)

def get_content(content_json):
    data = '''
    获取课程信息列表
    :param content_json:获取的json格式数据 
    :return: 课程数据
    '''
    if 'result' in content_json:
        return content_json['result']['list']  # 返回课程数据列表

def save_excel(content,index):
    data = '''
    将数据存储到excel表中
    :param content:课程内容 
    :param index: 索引值，从0开始
    :return: None
    '''
    for num,item in enumerate(content):
        # print(num,item)   # 打印课程信息
        row = 50 * index + (num + 1)
        worksheet.write(row, 0, item['productId'])
        worksheet.write(row, 1, item['courseId'])
        worksheet.write(row, 2, item['productName'])
        worksheet.write(row, 3, item['productType'])
        worksheet.write(row, 4, item['provider'])
        worksheet.write(row, 5, item['score'])
        worksheet.write(row, 6, item['scoreLevel'])
        worksheet.write(row, 7, item['learnerCount'])
        worksheet.write(row, 8, item['lessonCount'])
        worksheet.write(row, 9, item['lectorName'])
        worksheet.write(row, 10, item['originalPrice'])
        worksheet.write(row, 11, item['discountPrice'])
        worksheet.write(row, 12, item['discountRate'])
        worksheet.write(row, 13, item['imgUrl'])
        worksheet.write(row, 14, item['bigImgUrl'])
        worksheet.write(row, 15, item['description'])
def main(index):
    content_json = get_json(1)
    content = get_content(content_json)
    save_excel(content, index)
if __name__ == '__main__':
    print("开始执行")
    # 创建一个excel表格
    workbook = xlsxwriter.Workbook('网易云课堂python课堂数据.xlsx')
    worksheet = workbook.add_worksheet('first_sheet')
    worksheet.write(0, 0, '商品ID')
    worksheet.write(0, 1, '课程ID')
    worksheet.write(0, 2, '商品名称')
    worksheet.write(0, 3, '商品类型')
    worksheet.write(0, 4, '机构名称')
    worksheet.write(0, 5, '评分')
    worksheet.write(0, 6, '评分等级')
    worksheet.write(0, 7, '学习人数')
    worksheet.write(0, 8, '课程节数')
    worksheet.write(0, 9, '讲师名称')
    worksheet.write(0, 10, '原价')
    worksheet.write(0, 11, '折扣价')
    worksheet.write(0, 12, '折扣率')
    worksheet.write(0, 13, '课程小图URL')
    worksheet.write(0, 14, '课程大图URL')
    worksheet.write(0, 15, '课程描述')
    totlePageCount = get_json(1)['result']['query']['totlePageCount']# 获取总页数
    # 循环遍历每一页：
    for index in range(totlePageCount): # 9
        main(index)
    workbook.close()  # 关闭Excel的写入
    print("结束")

生成的excel文件

如果这篇文章对你有帮助的话，点个赞呗~