python爬虫下载电影百度文档_Python爬虫之《电影天堂》电影详情+下载地址爬取...

纪念自学Python以来，自己成功写出的第一个爬虫程序......

import requests

from lxml import etree

BASE_DOMAIN = 'http://dytt8.net'

HEADERS = {

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '

'Chrome/68.0.3440.106 Safari/537.36',

'Referer': 'http://dytt8.net/html/gndy/dyzz/list_23_2.html'

}

def get_detail_urls(url):

"""

获取电影天堂一个页面中的所有电影详情页面

:param url: 要获取的地址（电影天堂首页或第n页）

:return: 返回一个带有所有详情页面地址的列表

"""

# url = 'http://dytt8.net/html/gndy/dyzz/list_23_1.html'

response = requests.get(url, headers=HEADERS)

# gbk会报错，忽略错误进行解码

text = response.content.decode('gbk', "ignore")

html = etree.HTML(text)

details_urls = html.xpath("//table[@class='tbspan']//a/@href")

all_details_urls = []

for details_url in details_urls:

if 'index.html' in details_url:

pass

else:

link = BASE_DOMAIN + details_url

all_details_urls.append(link)

return all_details_urls

def parse_detail_page(url):

"""

解析电影详情页面的信息

:param url: 具体电影的详细地址

:return: 返回解析好的电影信息字典

"""

movies = {}

response = requests.get(url, headers=HEADERS)

text = response.content.decode('gbk', "ignore")

html = etree.HTML(text)

title = html.xpath("//h1/font[@color='#07519a']/text()")[0] # 获取标题

zoom_element = html.xpath("//div[@id='Zoom']")[0] # 获取到zoom元素

imgs = zoom_element.xpath(".//img/@src") # 获取海报和电影截图

movie_poster = imgs[0] # 获取到电影海报

if len(imgs) > 1:

movie_screenshot = zoom_element.xpath(".//img/@src")[1] # 获取到电影截图

else:

movie_screenshot = '没有电影截图'

movie_infos = zoom_element.xpath(".//text()")

movie_download_url = zoom_element.xpath(".//td[@bgcolor='#fdfddf']/a/@href")[0] # 获取下载链接

movies['电影名'] = title

movies['电影海报'] = movie_poster

movies['电影截图'] = movie_screenshot

for i, info in enumerate(movie_infos, 0):

if info.startswith('◎年　　代'):

info = info.replace('◎年　　代', '').strip() # str.strip()方法是去掉字符串头尾空格，正好用于这里。

movies['上映时间'] = info

elif info.startswith('◎产　　地'):

info = info.replace('◎产　　地', '').strip() # str.strip()方法是去掉字符串头尾空格，正好用于这里。

movies['电影产地'] = info

elif info.startswith('◎类　　别'):

info = info.replace('◎类　　别', '').strip() # str.strip()方法是去掉字符串头尾空格，正好用于这里。

movies['电影类型'] = info

elif info.startswith('◎豆瓣评分'):

info = info.replace('◎豆瓣评分', '').strip() # str.strip()方法是去掉字符串头尾空格，正好用于这里。

movies['电影评分'] = info

elif info.startswith('◎片　　长'):

info = info.replace('◎片　　长', '').strip() # str.strip()方法是去掉字符串头尾空格，正好用于这里。

movies['电影片长'] = info

elif info.startswith('◎导　　演'):

info = info.replace('◎导　　演', '').strip()

movies['电影导演'] = info

elif info.startswith('◎主　　演'):

starring = []

for x in range(i, len(movie_infos)):

if movie_infos[x].startswith('◎简　　介'): # 如果到了简介，就代表主演列表结束，那么就结束循环

break

starring.append(movie_infos[x].replace('◎主　　演', '').strip())

movies['电影主演'] = str.join('\n', starring) # 把主演列表通过join转换成字符串

elif info.startswith('◎简　　介'):

info = movie_infos[i + 1].strip() # 简介储存在当前下标的下一个位置

movies['电影简介'] = info

movies['迅雷下载'] = movie_download_url

return movies

def spider():

target_pages = input('请输入要爬取的页数：')

# 至少应该爬取第一页。

if target_pages.isdigit():

target_pages = int(target_pages)

if target_pages >= 2:

target_pages = target_pages + 1

else:

target_pages = 2

print(target_pages)

base_url = 'http://dytt8.net/html/gndy/dyzz/list_23_{}.html' # 使用占位符到时候用来确定是第几页

for i in range(1, target_pages):

print('=' * 80)

print('......当前正在爬取第{0}页，请稍后...... Spider is working......'.format(i))

print('=' * 80)

url = base_url.format(i)

detail_urls = get_detail_urls(url)

for detail_url in detail_urls:

movies = parse_detail_page(detail_url)

for key, value in movies.items():

print(key, value)

if __name__ == '__main__':

spider()