spa3(Ajax加载、爬虫练习)

spa3(Ajax加载、爬虫练习)

代码

# coding = utf-8
# @Time : 2021/12/25
# url = 'https://spa3.scrape.center/'
import requests
import json
import openpyxl


# 传入页码，返回每一页对应的的url
def get_index_url(offset):
    url = 'https://spa3.scrape.center/api/movie/?limit=10&offset={}'.format(offset)
    return url


# 发送请求，返回响应
def get_html(url):
    response = requests.get(url).text
    return response


# 解析数据
def parse_data(response):
    # page_data用于储存每一页的数据
    page_data = []
    # 将json字符串转换为Python对象
    res = json.loads(response)
    results = res['results']
    # 遍历每一部电影的数据，进行信息提取
    for result in results:
        page_dict = {}
        page_dict['name'] = result['name']
        # 因为后面要保存到excel文件，故这里预先处理一下
        page_dict['categories'] = '、'.join(result['categories'])
        page_dict['regions'] = '、'.join(result['regions'])
        page_dict['score'] = result['score']
        page_dict['abstract'] = result['drama']

        page_data.append(page_dict)

    return page_data


# 保存信息至excel
def save_data(page_data):
    # 打开excel文件
    wb = openpyxl.load_workbook('spa3.xlsx')
    ws = wb['Sheet']
    # 按行写入数据
    for data in page_data:
        ws.append([data['name'], data['categories'], data['regions'], data['score'], data['abstract']])
    wb.save('spa3.xlsx')


def main():
    # 创建excel文件,用于存储数据
    wb = openpyxl.Workbook()
    wb.save('spa3.xlsx')

    for page in range(3):
        print('正在提取第{}页，，，'.format(page+1))
        url = get_index_url(page*10)
        response = get_html(url)
        page_data = parse_data(response)
        # print(page_data)

        # 调用save_data方法，按页向excel写入数据
        print('正在保存第{}页，，，'.format(page+1))
        save_data(page_data)
        # print(page_data)
    print('信息保存完毕！')


if __name__ == '__main__':
    main()

结果