前言
想要做一个街舞网站视频的app,利用爬取的街舞视频存入本地,展现到app上
爬取网站:https://www.vhiphop.com/
Vhiphop网站也是我很喜欢的街舞网站之一,这里聚集了很多我喜欢的舞者视频和教学
爬取代码:
# !/usr/bin/env python
# -*- coding: utf-8 -*-
# 爬取Vhiphop网站
import requests
import importlib
import lxml.html
import os
import sys
importlib.reload(sys)
import json
from selenium import webdriver
video_list = []
def selenium_get_vhiphop_html(http_url):
# 更改代理ip
options = webdriver.ChromeOptions()
options.add_argument("--proxy-server=http://110.73.2.248:8123")
#获取chromedriver的绝对路径
driver_path = r'E:\fiddler\chromedriver.exe'
# 初始化一个driver,并且指定chromedriver的路径
driver = webdriver.Chrome(executable_path=driver_path)
# driver = webdriver.Chrome(executable_path=driver_path,, chrome_options=options)
# 请求网页
driver.get(http_url)
# 通过page_source获取网页源代码 热键 ctrl+alt+L
return driver.page_source
# request请求,但response内容隐藏
def parse_vhiphop_url(http_url):
#拿到请求头 模拟用户搜索
headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"}
response = requests.get(http_url,headers=headers,verify=False)
# 响应首页内容
return response.text
# 处理html数据,得到所有的数据
def get_vhiphop_video(html_data):
"""分析全网数据,并提取视频信息"""
# 获得etree对象
metree = lxml.html.etree
# 获得解析对象
parser = metree.HTML(html_data)
# 解析并获得div列表
li_list = parser.xpath("//ul[@class='video-lists-box']/li")
# 遍历div列表
for li_element in li_list:
# 拿到一个个li标签
video_title = li_element.xpath("./a[@class='video-lists-link']/@title")
# 40个popping信息
item = {}
# 获取标题
item["video_title"] = video_title[0]
# 获取ul地址,以用查找视频地址
video_data_html_url = li_element.xpath("./a[@class='video-lists-link']/@href")[0]
if not video_data_html_url.startswith("http"):
video_data_html_url = "https://www.vhiphop.com"+video_data_html_url
# 获取街舞视频播放页面html数据
# video_data = selenium_get_vhiphop_html(video_data_html_url)
# 解析html数据
video_url = video_real_url(video_data_html_url)
item["video_url"] = video_url
# video封面图片地址
video_img_url = li_element.xpath("./a[@class='video-lists-link']/div/img/@data-src")[0]
item["video_img_url"] = video_img_url
# 获取街舞类型:popping lacking breaking
video_type = li_element.xpath("./a[@class='video-lists-link']/div[@class='video-meta-box']/div[@class='video-title-box']/span/text()")[0]
item["video_type"] = video_type
video_list.append(item)
def selenium_get_video_html(html_url):
# 获取chromedriver的绝对路径
driver_path = r'E:\fiddler\chromedriver.exe'
# 初始化一个driver,并且指定chromedriver的路径
driver = webdriver.Chrome(executable_path=driver_path)
# driver = webdriver.Chrome(executable_path=driver_path,, chrome_options=options)
# 请求网页
driver.get(html_url)
# 通过page_source获取网页源代码 热键 ctrl+alt+L
return driver.page_source
def video_real_url(video_html_url):
html_data = selenium_get_video_html(video_html_url)
# 获得etree对象
metree = lxml.html.etree
# 获得解析对象
parser = metree.HTML(html_data)
# 解析并获得div
video_url = parser.xpath("//div[@class='video-container cp']/div/video[@class='vjs-tech']/@src")
return video_url
def result(http_url):
item = {}
html_data = selenium_get_vhiphop_html(http_url)
get_vhiphop_video(html_data)
def save_video_datas_json_file(video_list):
"""以json格式保存到本地"""
# 当目录不存在时则创建新目录
video_dir_name = "./videos"
if not os.path.exists(video_dir_name):
# 创建目录
os.makedirs(video_dir_name)
# 写入数据 python类型的数据转换为字符串
video_json_strs = json.dumps(video_list,ensure_ascii=False,indent=2)
# 写入文件
video_file = open(video_dir_name+"/video.json","w",encoding="utf-8")
video_file.write(video_json_strs)
video_file.close()
print("总共个数:"+len(video_list))
print("视频数据已经保存成功!谢谢!")
def main():
# 推荐页
http_url_index = "https://www.vhiphop.com/"
# choreograph = 1
# 编舞
http_url_choreograph = "https://www.vhiphop.com/videos?sort=1"
result(http_url_choreograph)
# popping
http_url_popping = "https://www.vhiphop.com/videos?sort=2"
result(http_url_popping)
# hiphop
http_url_hiphop = "https://www.vhiphop.com/videos?sort=3"
result(http_url_hiphop)
# jazz
http_url_jazz = "https://www.vhiphop.com/videos?sort=4"
result(http_url_jazz)
# locking
http_url_locking = "https://www.vhiphop.com/videos?sort=5"
result(http_url_locking)
# waccking
http_url_waccking = "https://www.vhiphop.com/videos?sort=6"
result(http_url_waccking)
# breaking
http_url_breaking = "https://www.vhiphop.com/videos?sort=7"
result(http_url_breaking)
# house
http_url_house = "https://www.vhiphop.com/videos?sort=8"
result(http_url_house)
# contemporary
http_url_contemporary = "https://www.vhiphop.com/videos?sort=9"
result(http_url_contemporary)
# reggae
http_url_reggae = "https://www.vhiphop.com/videos?sort=10"
result(http_url_reggae)
# freestly
http_url_freestly = "https://www.vhiphop.com/videos?sort=11"
result(http_url_freestly)
# krumping
http_url_krumping = "https://www.vhiphop.com/videos?sort=12"
result(http_url_krumping)
# others
http_url_others = "https://www.vhiphop.com/videos?sort=13"
result(http_url_others)
# 解析网址,请求数据
# [{视频1},{视频2},{视频3}]
# 保存文件
save_video_datas_json_file(video_list)
# item_popping = get_vhiphop_video(html_data_index)
if __name__ == '__main__':
main()
爬取的数据以json存入文本
response内容隐藏
request请求相应的response内容不全
检查网页元素可以看到 response内容被隐藏,可以通过selenium直接获取浏览器的Element
缺点
因为需要安装浏览器驱动,所以每次请求网页时都需要打开浏览器,加载时间实在太长。建议采用多线程爬取,等待时间缩短一些
后续优化
等我写好app了在搞