import time
from urllib.request import urlopen, Request
import requests
from lxml import etree
import re
from urllib import request
url = "https://movie.douban.com/top250?start=%s"
headers ={
'User-Agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36"
}
def getInfo(page):
print(f"正在打印第{page}页的信息")
req = request.Request(url%(page), headers=headers)
ret = request.urlopen(req).read().decode("utf-8")
#print(ret) 测试是否拿到html网页
html = etree.HTML(ret)
path = '//*[@id="content"]/div/div[1]/ol/li'
items = html.xpath(path)
title_path = 'div/div[2]/div[1]/a/span[1]/text()'
photo_path = 'div/div[1]/a/img/@src'
for i in items:
title = i.xpath(title_path)
print(title[0])
photo = i.xpath(photo_path)[0]
request.urlretrieve(photo,"d:/photo/"+title[0]+photo[photo.rfind("."):])
time.sleep(0.1)
print("-"*50)
if __name__ =="__main__":
for i in range(0,2):
getInfo(i)
time.sleep(1)
# 单个爬取
# title_path = '//*[@id="content"]/div/div[1]/ol/li[1]/div/div[2]/div[1]/a/span[1]/text()'
# photo_path = '//*[@id="content"]/div/div[1]/ol/li[1]/div/div[1]/a/img/@src'
# people_num = '//*[@id="content"]/div/div[1]/ol/li[1]/div/div[2]/div[2]/div/span[4]/text()'
# data_title = html.xpath(title_path)
# data_photo = html.xpath(photo_path)
# data_people = html.xpath(people_num)
# # data_people_num = re.sub("人评价$","",data_people)
#
# print(data_title,'-',data_photo,'-',data_people)
使用python简易代码爬取豆瓣top250电影名字标题和图片
最新推荐文章于 2022-10-21 19:39:41 发布