简单的利用xpath写的爬虫:
爬取电影天堂前七页电影的详细信息
需要注意的点就是 电影天堂网站的解码方式虽然标注为gbk,但某些字符网站解码也是乱码,在代码中很容易报错,所以直接使用etree.HTML(text)生成对象做处理就好,不用做一些解码操作
思路
爬取全部电影页面—>获得详细电影页面的url—>爬取详细电影信息界面—>利用xpath数据筛选处理
import requests
from lxml import etree
BASE_DOMAIN = ' https://www.dytt8.net'
HEADERS ={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36'
}
#获得电影所在详细信息的url
def get_detail_urls(url):
response = requests.get(url, headers=HEADERS)
text = response.content
htmlelement = etree.HTML(text)
detail_urls = htmlelement.xpath("//table[@class='tbspan']//a/@href")
detail_urls = map(lambda url:BASE_DOMAIN+url,detail_urls)
return detail_urls
#调用其余两个方法实现目的
def spider():
base_url=" https://www.dytt8.net/html/gndy/dyzz/list_23_{}.html"
movies=[]
#依次爬取前7页
for x in range(1,8):
url = base_url.format(x)
detail_urls = get_detail_urls(url)
print(detail_urls)
#根据获得的详细url爬取详细信息
for detail_url in detail_urls:
movie=parse_detail_page(detail_url)
movies.append(movie)
print(movies)
#爬取详情页面,并对爬取的数据进行处理
def parse_detail_page(url):
movie = {}
response = requests.get(url,headers = HEADERS)
text = response.content.decode('gbk')
htmlelement = etree.HTML(text)
title = htmlelement.xpath("//div[@class='title_all']//font[@color='#07519a']/text()")[0]
movie['title'] = title
zoom = htmlelement.xpath("//div[@id='Zoom']")[0]
cover = zoom.xpath(".//img/@src")[0]
movie['cover']= cover
infos = zoom.xpath('.//text()')
for info in infos:
if info.startswith("◎年 代"):
info = info.replace("◎年 代","").strip()
movie['year'] = info
elif info.startswith("◎类 别"):
info = info.replace("◎类 别","").strip()
movie['type'] = info
return movie
if __name__ == '__main__':
spider()