Python爬取豆瓣正在上映的电影
#爬取豆瓣正在上映的电影
import requests
from lxml import etree
#1、将目标从网站上的页面抓取下来
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36',
'Referer':'https://movie.douban.com/'
}
url = 'https://movie.douban.com/cinema/nowplaying/dalian/'
response = requests.get(url,headers=headers)
text = response.text
#2、将抓取下来的数据按一定的规则进行提取
movies = []
html = etree.HTML(text)
ul = html.xpath("//ul[@class='lists']")[1]
lis = ul.xpath("./li")
for li in lis:
title = li.xpath("@data-title")[0]
score = li.xpath("@data-score")[0]
release = li.xpath("@data-release")[0]
duration = li.xpath("@data-duration")[0]
region = li.xpath("@data-region")[0]
director = li.xpath("@data-director")[0]
actors = li.xpath("@data-actors")[0]
movie = {
'title':title,
'score': score,
'release': release,
'duration': duration,
'region': region,
'director': director,
'actors': actors
}
movies.append(movie)
print(movies)