记录一个爬取电影天堂的爬虫

import requests

from lxml import etree

BASE_DOMAIN = ' https://www.dytt8.net'
HEADERS ={
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36'
}
#获得电影所在详细信息的url
def get_detail_urls(url):
    response = requests.get(url, headers=HEADERS)
    text = response.content
    htmlelement = etree.HTML(text)
    detail_urls = htmlelement.xpath("//table[@class='tbspan']//a/@href")
    detail_urls = map(lambda url:BASE_DOMAIN+url,detail_urls)
    return detail_urls
#调用其余两个方法实现目的
def spider():
    base_url=" https://www.dytt8.net/html/gndy/dyzz/list_23_{}.html"
    movies=[]
    #依次爬取前7页
    for x in range(1,8):
        url = base_url.format(x)
        detail_urls = get_detail_urls(url)
        print(detail_urls)
        #根据获得的详细url爬取详细信息
        for detail_url in detail_urls:
            
            movie=parse_detail_page(detail_url)
            movies.append(movie)
  	print(movies)

#爬取详情页面，并对爬取的数据进行处理
def parse_detail_page(url):
    movie = {}
    response = requests.get(url,headers = HEADERS)
    text = response.content.decode('gbk')
    htmlelement = etree.HTML(text)
    title = htmlelement.xpath("//div[@class='title_all']//font[@color='#07519a']/text()")[0]
    movie['title'] = title
    
    zoom = htmlelement.xpath("//div[@id='Zoom']")[0]
    
    cover = zoom.xpath(".//img/@src")[0]
    movie['cover']= cover
    
    infos = zoom.xpath('.//text()')
    for info in infos:
    	if info.startswith("◎年　　代"):
             info = info.replace("◎年　　代","").strip()
             movie['year'] = info
        elif info.startswith("◎类　　别"):
             info = info.replace("◎类　　别","").strip()
             movie['type'] = info
    return movie





if __name__ == '__main__':
    spider()