【无标题】

爬取豆瓣top250

import requests
from bs4 import BeautifulSoup
#访问top250主页
headers = {'user-agent':'my-app/0.0.1'}
#跳转页面 ?start&filter=
#先访问url链接，加上参数0，25，50，75~~~225
page = 0
max_page = 225#第十页的start 值
movie_links = []
movie_names = []
while page<=max_page:
    #访问页面
    url = "https://movie.douban.com/top250?start="+page.__str__()+"&filter="#这里的page后面跟的是转换为string类型！！注意一下
    reponse = requests.get(url=url,headers=headers)
    #print(reponse.text)
#实现每个页面的抓取:电影单链
    soup = BeautifulSoup(reponse.text,'html.parser')
    #print(soup.find_all(class_="hd"))
    for ele in soup.find_all(class_="hd"):
        #print(ele.find(class_="title").string)
        #print(ele)
        #print(ele.find('a',href=True).attrs['href'])
        movie_names.append(ele.find(class_="title").string)
        movie_links.append(ele.find('a',href=True).attrs['href'])
# #修改start参数
    page +=25
#验证数据正确性
    print(url)
    #退出
    #exit()
#浏览所有抓取到的信息
for name,link in zip(movie_names,movie_links):
    print(name,":",link)
print(len(movie_names))