基础爬虫案例

1. 爬取数据并保存为 txt 文件

import requests
import json
# 获取douban电影网址
url = 'https://movie.douban.com/j/chart/top_list'
# 获取 headers
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36'
}
# 获取参数，type 决定了是哪种类型的电影，可以根据需要修改
params = {
'type': '11',
'interval_id': '100:90',
'action':'', 
'start': '0',
'limit': '10',
}
# resp = requests.get(url, verify=False)  # verify = False  去掉安全验证
response = requests.get(url=url, headers=headers, params=params)
content = response.json()

with open('douban电影喜剧评分排行榜.txt', 'w', encoding='utf-8') as fp:
    for i in content:
        title = i['title']
        score = i['score']
        regions = i['regions']
        fp.write(title+' '+score+' '+str(regions)+'\n')

print('爬虫结束！！！')

2. 保存为表格并输出 excel 文件

import pandas as pd
a = {'title':[],
    'score':[],
    'regions':[]}
for i in content:
    a['title'].append(i['title'])
    a['score'].append(i['score'])
    a['regions'].append(i['regions'])
    
df = pd.DataFrame(a)
# df.head()

df.to_csv(r'电影评分.csv', encoding='utf-8')