来张图片压压惊!!!1
爬取电影天堂的电影信息。
from lxml import etree
import requests
BASE = 'https://www.dytt8.net'
HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36',
'Referer': 'https://www.dytt8.net/html/gndy/dyzz/list_23_2.html'
}
def get_urls(url):
response = requests.get(url,headers=HEADERS) #获取网页
text = response.text #解码后是str类型
html = etree.HTML(text)
table_urls = html.xpath("//table[@class='tbspan']//a/@href")
table_urls = map(lambda url: BASE + url,table_urls)#使用lambda表达式进行网址链接
return table_urls
def parse_page(url):
movie = {}
response = requests.get(url,headers=HEADERS)
text = response.content.decode('gbk')
html = etree.HTML(text)
title = html.xpath("//div[@class='title_all']//font[@color='#07519a']//text()")[0] #获取标签下的所有文本
# print(title)
movie['title'] = title
zoomA = html.xpath('//div[@id="Zoom"]')[0]
imgs = zoomA.xpath(".//img/@src")
# print(imgs)
cover = imgs[0] #获取海报
# print(cover)
#screenshot = imgs[1] #获取截屏
movie['cover'] = cover
#movie['screenshot'] = screenshot
def parse_info(info,rule):
return info.replace(rule,"").strip() #用strip()函数将字段两边的空格去掉 replace()函数将年代下面的字段替换成空
infos = zoomA.xpath(".//text()") #获取当前下的所有文本
for index,info in enumerate(infos): #enumerate() 函数用于将一个可遍历的数据对象(如列表、元组或字符串)组合为一个索引序列,返回索引和数据
# print(index)
# print(info)
# print("="*10)
if info.startswith("◎年 代"): #字段的起始位置◎年 代
info=parse_info(info,"◎年 代")
# print(info)
movie['year'] = info
elif info.startswith("◎产 地"):
info=parse_info(info,"◎产 地")
movie['country'] = info
elif info.startswith("◎类 别"):
info=parse_info(info,"◎类 别")
movie['category'] = info
elif info.startswith("◎豆瓣评分"):
info=parse_info(info,"◎豆瓣评分")
movie['douban_rating'] = info
elif info.startswith("◎片 长"):
info=parse_info(info,"◎片 长")
movie['duration'] = info
elif info.startswith("◎导 演"):
info=parse_info(info,"◎导 演")
movie['director'] = info
elif info.startswith("◎主 演"):
info=parse_info(info,"◎主 演")
actors = [info] #先将上面的主演的第一个放到 actors
for i in range(index+1,len(infos)):
actor = infos[i].strip() #遍历所有的主演 去掉两边的空格
# print(actor)
if actor.startswith("◎"): # 取到这就终止
break
actors.append(actor)
movie["actors"] = actors
elif info.startswith("◎简 介"):
info=parse_info(info,"◎简 介")
for x in range(index+1,len(infos)):
profile = infos[x].strip()
movie["profile"]=profile
download_url = html.xpath('//td[@bgcolor="#fdfddf"]/a/@href')[0]
# print(download_url)
movie["download_url"] = download_url
return movie
def spider():
base_url = "https://www.dytt8.net/html/gndy/dyzz/list_23_{}.html"
movies = []
for i in range(1,8): #获取前 7 页的页面
url = base_url.format(i) #每个页面的所有电影网址
tables = get_urls(url)
for table in tables: #具体的电影网址
movie = parse_page(table)
movies.append(movie)
# print(movie)
with open('D:/pachong/天堂movie.json','w',encoding='utf-8') as f:
for i in movies:
ii = str(i) + '\n'
f.write(ii)
print("下载完成")
if __name__ == "__main__":
spider()