爬取熊猫平台的数据也是使用面向对象的思想,和同样的逻辑思维,可以借鉴一下这种逻辑思维。至于解析可以参看我的这一篇博客:https://blog.csdn.net/qq_39198486/article/details/82950583
如果使用seleinum模块时不会配置chromedriver文件,可以参考这篇博客:https://blog.csdn.net/qq_39198486/article/details/82930025
下面我就直接放全部代码,主要地方我都有注释,就不一一在代码外写出来了:
# author: aspiring
from selenium import webdriver
import time
import json
class XiongmaoSpider:
def __init__(self):
self.start_url = "https://www.panda.tv/all" # start_url
self.driver = webdriver.Chrome() # 实例化一个浏览器
def get_content_list(self): # 提取数据
li_list = self.driver.find_elements_by_xpath("//ul[@id='later-play-list']/li") # 分组
content_list = []
for li in li_list:
item = {}
item["name"] = li.find_element_by_xpath(".//span[@class='video-nickname']").get_attribute("title")
item["title"] = li.find_element_by_xpath(".//span[@class='video-title']").text
item["room_img"] = li.find_element_by_xpath(".//img[@class='video-img video-img-lazy']").get_attribute("data-original")
item["watch_num"] = li.find_element_by_xpath(".//span[@class='video-number']").text
print(item)
content_list.append(item) # 将字典放入逐条添加到一个列表内
# 获取下一页元素
next_url = self.driver.find_elements_by_xpath("//a[@class='j-page-next']")
# 确保获取最后一页的出现没有下一页时不会报错,并在while循环中作为判别条件
next_url = next_url[0] if len(next_url) > 0 else None
return content_list, next_url
def save_content_list(self, content_list):
with open("xiongmao.txt", "a", encoding="utf-8") as f:
for content in content_list:
f.write(json.dumps(content, ensure_ascii=False, indent=2)) # 使用json将数据以json格式写入文件
f.write("\n")
def run(self): # 实现主要逻辑
# 1.start_url
# 2.发送请求,获取响应
self.driver.get(self.start_url)
# 3.提取数据
content_list, next_url = self.get_content_list()
# 4.保存
self.save_content_list(content_list)
# 点击下一页元素
while next_url is not None:
next_url.click() # 点击下一页
time.sleep(2) # 睡2s是为了下一页元素的载入缓冲时间,防止页面元素还没加载出来就去提取数据
# 3.提取数据
content_list, next_url = self.get_content_list()
# 4.保存
self.save_content_list(content_list)
if __name__ == '__main__':
xiongmao = XiongmaoSpider()
xiongmao.run()
下面是导出的json格式的文件,我列举l前三个数据:
{
"name": "沐慈Kiki",
"title": "Happy day香槟 啤酒抽奖",
"room_img": "https://i.h2.pdim.gs/90/c0a8df56c6462e3882782f4fc22602ff/w338/h190.jpg",
"watch_num": "1.3万"
}
{
"name": "芒果鱼丶",
"title": "韩服大师上王者",
"room_img": "https://i.h2.pdim.gs/90/9c28dff6ad3d6bf1cef25e9062c0257e/w338/h190.jpg",
"watch_num": "19.4万"
}
{
"name": "会旋转的冬瓜丶",
"title": "瓜式一刀流 开斩!",
"room_img": "https://i.h2.pdim.gs/90/9fdc00fe5a2b9252765468ff2cd533dd/w338/h190.jpg",
"watch_num": "18.0万"
}
...
...