Python_爬虫_requests和bs4

#_爬虫_豆瓣TOP电影排行榜

import requests     # 爬取网页
import bs4      # 检索解析数据

host = 'https://movie.douban.com/top250'        # 网址


# 找出一共有几个页面
def find_depth(res):
    soup = bs4.BeautifulSoup(res.text, 'html.parser')
    depth = soup.find('span', class_='next').previous_sibling.previous_sibling.text
    return int(depth)


# 解析网页内容
res = requests.get(host)
depth = find_depth(res)

result = []
for i in range(depth):
    url = host + '?start=' + str(25*i) + '&filter='        # 找下一个网址和上一个网址的规律
    res = requests.get(url)
    soup = bs4.BeautifulSoup(res.text, 'html.parser')
    targets = soup.find_all('div', class_='hd')
    for each in targets:
        # print(each.a.span.text)
        # result.extend(each.a.span.text)
        result.append(each.a.span.text + '\n')     # 把每一条添加进结果


with open("豆瓣TOP250电影.txt", "w", encoding="utf-8") as f:        # 在同一个文件夹下，创建豆瓣TOP250电影.txt，并写入数据
    for each in result:
        f.write(each)

if __name__ == "__main__":      # 测试使用，如果调用程序则不运行
    for each in result:
        print(each)