python3-爬虫之电影天堂

pyhon3爬虫之——电影天堂最新电影信息

#导入所需要的库
import requests
from lxml import etree

定义全局变量
BASE_URL="https://www.dytt8.net"
HEADERS = {
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"
    }

#解析最新电影列表页面，获取当前页面每个电影的url
def get_detil_urls(url):
    response = requests.get(url, headers=HEADERS)
    text = response.text
    html = etree.HTML(text)
    detail_urls = html.xpath('//table[@class="tbspan"]//a/@href')
    # for deteil_url in deteil_urls:
    #     print(BASE_URL + deteil_url)
    detail_urls=map(lambda url:BASE_URL+url,detail_urls)
    return detail_urls

#解析电影详情页面，获取电影详情信息
def parse_detail_page(url):
    movie={}
    response=requests.get(url,headers=HEADERS)
    text=response.content.decode("gbk")
    html=etree.HTML(text)
    title=html.xpath("//div[@class='title_all']//font/text()")[0]
    movie["title"]=title

    zoomE=html.xpath("//div[@id='Zoom']")[0]
    imgs=zoomE.xpath(".//img/@src")
    cover=imgs[0]
    screenshot=imgs[1]
    movie["cover"]=cover
    movie["screenshot"]=screenshot

    def parse_info(info,rule):
        return info.replace(rule,"").strip()
    infos=zoomE.xpath(".//text()")
    for index,info in enumerate(infos):
        if info.startswith("◎年　　代"):
            info=parse_info(info,"◎年　　代")
            movie['year']=info
        elif info.startswith("◎产　　地"):
            info=parse_info(info,"◎产　　地")
            movie["country"]=info
        elif info.startswith("◎类　　别"):
            info=parse_info(info,"◎类　　别")
            movie["category"]=info
        elif info.startswith("◎豆瓣评分"):
            info=parse_info(info,"◎豆瓣评分")
            movie["score"]=info
        elif info.startswith("◎片　　长"):
            info=parse_info(info,"◎片　　长")
            movie["duration"]=info
        elif info.startswith("◎导　　演"):
            info=parse_info(info,"◎导　　演")
            movie["director"]=info
        elif info.startswith("◎主　　演"):
            info=parse_info(info,"◎主　　演")
            actors=[info]
            for x in range(index+1,len(infos)):
                actor=infos[x].strip()
                if actor.startswith("◎"):
                    break
                actors.append(actor)
            movie["actors"]=actors
        elif info.startswith("◎简　　介"):
            info=parse_info(info,"◎简　　介")
            text=[]
            for x in range(index+1,len(infos)):
                profile=infos[x].strip()
                if profile.startswith("【下载地址】"):
                    break
                text.append(profile)
            movie['profile']=text[0]

    download_url=html.xpath('//td[@bgcolor="#fdfddf"]/a/@href')
    movie["download_url"]=download_url
    return movie

def spider(n):
    base_url = "https://www.dytt8.net/html/gndy/dyzz/list_23_{}.html"
    movies=[]
    for x in range(1,n):
        #第一个for循环是用来控制总共有几页
        url=base_url.format(x)
        detail_urls=get_detil_urls(url)
        for detail_url in detail_urls:
            #第二个for循环是用来遍历每一页的所有电影的详情url
            movie=parse_detail_page(detail_url)
            movies.append(movie)
            print(movie)

if __name__ =="__main__":
    spider()