from lxml import etree
import requests
import time
import csv
class Spider1():
# 发送请求,获取响应
def get_html(self, url):
response1 = requests.get(url, headers=self.headers, auth=("admin", "admin"))
time.sleep(2)
return response1.content.decode()
# 从响应中提取数据
def get_content(self, html_str):
html = etree.HTML(html_str)
div_list = html.xpath('//div[@class="el-col el-col-18 el-col-offset-3"]/div')
move_infos = []
for li in div_list:
name = li.xpath('.//div/div/div[2]/a/h2/text()')[0]
addr = li.xpath('.//div[@class="el-row"]/div[2]/div[2]/span[1]/text()')[0]
time_long = li.xpath('.//div[@class="el-row"]/div[2]/div[2]/span[3]/text()')[0]
time_agen = li.xpath('.//div[@class="el-row"]/div[2]/div[3]/span/text()')[0] if len(
li.xpath('.//div[@class="el-row"]/div[2]/div[3]/span/text()')) > 0 else None
img_url = li.xpath('.//div[@class="el-row"]/div[1]/a/img/@src')[0]
# 详情页
detail_url = 'https://ssr1.scrape.center' + li.xpath('./div/div/div[1]/a/@href')[0]
responses1 = requests.get(detail_url, headers=self.headers)
html = etree.HTML(responses1.content.decode())
datas = html.xpath('//div[@class="el-card__body"]/div/div[2]/div[4]/p/text()')[0]
move_info = [name, addr, time_long, time_agen, img_url, datas]
move_infos.append(move_info)
# 图片
filepath = r'./dyimg/' + name + ".png"
response2 = requests.get(img_url, headers=self.headers)
with open(filepath, "wb") as f:
f.write(response2.content)
return move_infos
def save_content(self, move_infos):
for i in move_infos:
self.w.writerow(i)
def run(self):
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 SLBrowser/9.0.0.10191 SLBChan/105'}
move_title = ["电影名", "出产地", "电影时长", "上映时间", "封面网址"]
with open("moves.csv", "w", encoding="utf_8", newline="") as f:
self.w = csv.writer(f)
self.w.writerow(move_title)
for i in range(1, 3):
url = "https://ssr3.scrape.center" + f"/page/{i}"
print(url)
html_str = self.get_html(url)
movies = self.get_content(html_str)
self.save_content(movies)
if __name__ == '__main__':
st = Spider1() # 实例化对象
st.run()
requests 用类方法爬取数据
最新推荐文章于 2024-09-03 17:37:40 发布
![](https://img-home.csdnimg.cn/images/20240711042549.png)