python爬取豆瓣排行榜电影(静态爬取)(二次爬取)

python爬取豆瓣排行榜电影(静态爬取)

爬取页面上每个电影的名称、年份，评分和类型。

获取网站url

因为我们爬取的内容第一个页面没有，需要点击电影进入详情

所以我们需要获取每个电影链接的url
打开f12寻找标签

url = 'https://movie.douban.com/chart'
r = requests.get(url,headers=headers)
r_b = BeautifulSoup(r.text,'lxml')
a_list = r_b.find_all(class_='pl2')#找到所有class为pl2的标签，每个电影所在html中的标签
url_list = []
for a in a_list:
    url_list.append(a.find('a').attrs['href'])#获取标签a下和链接

获取网站headers

按下f12，选中Network,按下f5刷新，

找到Cookie、referer、user-Agent
设置headers

headers = {
    'Cookie':'bid=n_IZTjBerIQ; __yadk_uid=ycqCriKvT0a5PcKLRNhceCyhO4Sz0Fah; ll="108307"; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1575852775%2C%22https%3A%2F%2Fwww.google.com%2F%22%5D; _pk_ses.100001.4cf6=*; ap_v=0,6.0; __utma=30149280.1355054136.1575426113.1575442804.1575852775.3; __utmb=30149280.0.10.1575852775; __utmc=30149280; __utmz=30149280.1575852775.3.2.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); __utma=223695111.1771316839.1575426113.1575442804.1575852776.3; __utmb=223695111.0.10.1575852776; __utmc=223695111; __utmz=223695111.1575852776.3.2.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); _pk_id.100001.4cf6=b00d2b706551734b.1575426113.3.1575852779.1575442796.',
    'referer':'https://movie.douban.com/',
    'user-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36'
}

get请求访问url

get访问每个电影的详情页url

for url_ in url_list:
    #print(url_)
    r_url = requests.get(url_,headers=headers)#访问每一个电影详情

BeautifulSoup解析网站

for url_ in url_list:
    #print(url_)
    r_url = requests.get(url_,headers=headers)#访问每一个电影详情
    r_url_b = BeautifulSoup(r_url.text,'lxml')

爬取html数据

    movie_title = r_url_b.h1.span.string#获取h1中span的内容(电影标题)
    time_ = r_url_b.h1.find('span',class_='year').string#找到class为year的标签span的内容(年份)
    movie_score = r_url_b.find('strong',class_='ll rating_num').string#找到class为ll rating_num的标签span的内容(电影评分)
    movie_types = r_url_b.find_all('span',property='v:genre')#找到property为v:genre的标签span的内容，类型有多种所以用find_all
    #info = r_url_b.find('div',id='info').text#找到id为info的div标签text
    movie_type_list = []
    movie_comment = r_url_b.find('span',property='v:summary')#找到preperty为v:summary的span标签(评论)
    for movie_type in movie_types:
        movie_type_list.append(movie_type.string)#获取每一种类型的值
    print(movie_title,time_,movie_score,movie_type_list)
    #print(info)
    print(movie_comment.text.replace(' ',''))

完整代码

import requests
from bs4 import BeautifulSoup
url = 'https://movie.douban.com/chart'
headers = {
    'Cookie':'bid=n_IZTjBerIQ; __yadk_uid=ycqCriKvT0a5PcKLRNhceCyhO4Sz0Fah; ll="108307"; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1575852775%2C%22https%3A%2F%2Fwww.google.com%2F%22%5D; _pk_ses.100001.4cf6=*; ap_v=0,6.0; __utma=30149280.1355054136.1575426113.1575442804.1575852775.3; __utmb=30149280.0.10.1575852775; __utmc=30149280; __utmz=30149280.1575852775.3.2.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); __utma=223695111.1771316839.1575426113.1575442804.1575852776.3; __utmb=223695111.0.10.1575852776; __utmc=223695111; __utmz=223695111.1575852776.3.2.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); _pk_id.100001.4cf6=b00d2b706551734b.1575426113.3.1575852779.1575442796.',
    'referer':'https://movie.douban.com/',
    'user-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36'
}
r = requests.get(url,headers=headers)
r_b = BeautifulSoup(r.text,'lxml')
a_list = r_b.find_all(class_='pl2')#找到所有class为pl2的标签
url_list = []
for a in a_list:
    url_list.append(a.find('a').attrs['href'])#获取标签a下和链接
#print(url_list)
for url_ in url_list:
    #print(url_)
    r_url = requests.get(url_,headers=headers)#访问每一个电影详情
    r_url_b = BeautifulSoup(r_url.text,'lxml')
    movie_title = r_url_b.h1.span.string#获取h1中span的内容(电影标题)
    time_ = r_url_b.h1.find('span',class_='year').string#找到class为year的标签span的内容(年份)
    movie_score = r_url_b.find('strong',class_='ll rating_num').string#找到class为ll rating_num的标签span的内容(电影评分)
    movie_types = r_url_b.find_all('span',property='v:genre')#找到property为v:genre的标签span的内容，类型有多种所以用find_all
    #info = r_url_b.find('div',id='info').text#找到id为info的div标签text
    movie_type_list = []
    movie_comment = r_url_b.find('span',property='v:summary')#找到preperty为v:summary的span标签(评论)
    for movie_type in movie_types:
        movie_type_list.append(movie_type.string)#获取每一种类型的值
    print(movie_title,time_,movie_score,movie_type_list)
    #print(info)
    print(movie_comment.text.replace(' ',''))