爬取豆瓣高分电影

用python爬取豆瓣高分电影 ——（来自b站up主的视频教程）

爬虫视频链接（因为原网站有变化，程序稍作修改，现在可以直接运行）

• 提前需要安装 requests 和 beautifulsoup

import requests
import bs4

def open_url(url):
    #使用代理 proxies(计算机科学技术 代理服务)
    # proxies={"http":"127.0.1:1080","https":"127.0.0.1:1080"}
    headers={'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134'}
    #res = requests.get(url,headers=hearders,proxies=proxies)
    res=requests.get(url,headers=headers)

    return res

def find_movies(res):
    soup=bs4.BeautifulSoup(res.text,'html.parser')
    #电影名
    movies=[]
    targets=soup.find_all("div",class_="hd")
    for each in targets:
        movies.append(each.a.span.text)
    
    #评分
    ranks=[]
    targets=soup.find_all("sapn",class_="rating_num")
    print(targets)
    for each in targets:
        ranks.append('评分：%s'%each.text)
    
    #资料
    messages = []
    targets = soup.find_all("div", class_="bd")
    for each in targets:
        try:
            messages.append(each.p.text.split(
                '\n')[1].strip()+each.p.text.split('\n')[2].strip())
        except:
            continue

    result=[]
    length=len(movies)

    for i in range(length):
        result.append(movies[i]+'\n' + messages[i]+'\n' )
        
    return result

#找出一共有多少个页面
def find_depth(res):
    soup=bs4.BeautifulSoup(res.text,'html.parser')
    depth=soup.find('span',class_='next').previous_sibling.previous_sibling.text

    return int(depth)

def main():
    host="http://movie.douban.com/top250"

    res = open_url(host)

    depth=find_depth(res)

    result=[]
    for i in range(depth):
        url = host+'/?start='+str(25*i)
        res = open_url(url)
        result.extend(find_movies(res))

    with open("豆瓣top250电影.txt","w",encoding="utf-8") as file:
       for each in result:
           file.write(each) 

if __name__=="__main__":
    main()

以上就是完整代码了，我是个python基础都没有学会的小菜鸡，先跟着网上的大佬视频，照着他们的思考的习惯，和代码风格，一点点学习，希望有朝一日也能成为一个有能力的人。加油努力！