import requests
from lxml import etree
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0'}
base_url = 'https://movie.douban.com/top250?start='
for page in range(150,226,25):
url = base_url + str(page)
response = requests.get(url, headers=headers, timeout=200000)
response.encoding = 'utf-8'
if response.status_code == 200:
if "资源不存在或已经删除" in response.text:
continue
else:
html = etree.HTML(response.text)
href = html.xpath('//a[@class=""]/@href')
for i in href:
res = requests.get(i, headers=headers, timeout=200000)
res.encoding = 'utf-8'
h = etree.HTML(res.text)
movie_top = h.xpath('//span[@class="top250-no"]//text()')
movie_name = h.xpath('//span[@property="v:itemreviewed"]//text()')
movie_type = h.xpath('//div[@id="info"]//span[@property="v:genre"]//text()')
movie_place = h.xpath('//div[@id="info"]//br[4]/text()')
movie_uptime = h.xpath('//span[@property="v:initialReleaseDate"]//text()')
movie_minutes = h.xpath('//span[@property="v:runtime"]/@content')
score = h.xpath('//strong[@class="ll rating_num"]//text()')
types = ""
times = ""
for j in movie_type:
types += j
types += "/"
types=types.strip("/")
for k in movie_uptime:
times += k
times += "/"
times=times.strip("/")
for (a, b, c, d) in zip(movie_top, movie_name, movie_minutes, score):
print(a + "," + b + "," + types + "," + times + "," + c + "," + d)
with open(r'F:\比赛--爬虫\douban3.txt', 'a', encoding='utf-8') as f:
# f.write(a+"\t"+b+"\t"+types+"\t"+times+"\t"+c+"\t"+d)
f.write(a + "," + b + "," + types + "," + times + "," + c + "," + d)
f.write("\n")
else:
continue
top250豆瓣电影爬取
最新推荐文章于 2024-09-12 16:43:24 发布