#Python&豆瓣&翻页

案例

import requests
import re
# url = "https://movie.douban.com/top250"
# csv 是表示数据与逗号隔开
f = open("top250.csv",mode='w',encoding='utf-8')

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36"
}


resp = requests.get("https://movie.douban.com/top250",headers = headers)
resp.encoding = "utf-8" # 解决页面乱码问题
pageSource = resp.text
# print(pageSource) 　# 验证数据，是否能不能拿到想要的数据

# 编写正则表达式
# re.S 可以让正则中的.匹配换行符
obj = re.compile(r'<div class="item">.*?<span class="title">(?P<name>.*?)</sp'
                 r'an>.*?<p class="">.*?导演:(?P<dao>.*?)&nbsp;.*?<br>'
                 r'(?P<year>.*?)&nbsp;.*?<span class="rating_num" property="v:average">'
                 r'(?P<score>.*?)</span>.*?<span>(?P<num>.*?)人评价</span>',re.S)

# 进行正则匹配
result = obj.finditer(pageSource)
for item in result:
    # print(item.group("name")) # 拿结果
    name = item.group("name")
    dao = item.group("dao")
    year = item.group("year").strip() # 去掉字符串左右两端的空白(空格)
    score = item.group("score")
    num = item.group("num")
    f.write(f"{name},{dao},{year},{score},{num}\n") # 如果觉着low,可以更换成csv模块，进行数据写入
    # print(name,dao,year,score,num)

f.close()    
resp.close()
print("doubanTOP250提取完毕.")

运行结果及图片展示

如图：

# 如何翻页提取？
# (页数 - 1)*25 => 25 start
# 翻页爬取doubanTOP250
for page in range(0,250,25):
    url = 'https://movie.douban.com/top250?start=' + str(page) + '&filter='
    resp = requests.get(url, headers=headers)
    resp.encoding = "utf-8" # 解决页面乱码问题
    pageSource = resp.text
    obj = re.compile(r'<div class="item">.*?<span class="title">(?P<name>.*?)</sp'
                         r'an>.*?<p class="">.*?导演:(?P<dao>.*?)&nbsp;.*?<br>'
                         r'(?P<year>.*?)&nbsp;.*?<span class="rating_num" property="v:average">'
                         r'(?P<score>.*?)</span>.*?<span>(?P<num>.*?)人评价</span>',re.S)
    result = obj.finditer(pageSource)
    for item in result:
        name = item.group("name")
        dao = item.group("dao")
        year = item.group("year").strip() # 去掉字符串左右两端的空白(空格)
        score = item.group("score")
        num = item.group("num")
        f.write(f"电影：{name},导演：{dao},年份：{year},评分：{score},评分：{num}\n") # 如果觉着low,可以更换成csv模块，进行数据写入
        # print(name,dao,year,score,num)

f.close()
resp.close()
print("doubanTOP250提取完毕.")

运行结果：