使用Java爬虫
Java使用jsoup爬取b站视频,根据模糊搜索,搜到批量视频自动下载。
首先先获取到b站的search链接,https://search.bilibili.com/all?keyword= keyword一般是elk的搜索引擎索引。
思路:
- 首先是使用jsoup爬取到b站的动态网页下来
-
String str=""; Scanner scanner = new Scanner(System.in); System.out.println("请输入想要搜索的内容"); str=scanner.next(); Document connect = null; try { connect = Jsoup.connect("https://search.bilibili.com/all?keyword="+str).get(); } catch (IOException e) { throw new RuntimeException(e); } Elements li = connect.getElementsByTag("li"); for (Element element : li) { String url = element.getElementsByTag("a").attr("href"); if (url.contains("www.bilibili.com/video/BV")){ list.add(url.substring(url.indexOf("BV")+2,url.indexOf("?"))); } }
- 获取到搜索到的视频网页,根据标签来获取他们的链接,获取到链接之后我们拿到链接里面的BV号,然后使用BV号去一个地址拿到这个视频的cid。
-
那到BV号之后我们使用get请求 请求这个"https://api.bilibili.com/x/web-interface/view?bvid="+bvid;地址就会返回一个json回来,我们获取到这个json里面的cid。
获取到这个cid之后,我们就是可以根据这个cid,去拿到这个视频的真实地址以及视频的真实信息
使用cid和BV号去请求这个地址 https://api.bilibili.com/x/player/playurl?cid=" + cid + “&fnver=0&qn=” + qn + “&otype=json&bvid=” + bvid + "&fnval=2&player=1 会获取到一个json
这里面的qn参数是对应视频的清晰度的一个参数 //qn : 视频质量 112 -> 高清 1080P+, 80 -> 高清 1080P, 64 -> 高清 720P, 32 -> 清晰 480P, 16 -> 流畅 360P // 最高支持 1080p, 1080P+是不支持的
url就是视频的真是路径,然后使用流下载下来到电脑的固定位置
下面是完整代码
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Map;
import java.util.Scanner;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class AppUrlMovie {
private static ExecutorService cacheExecutor = Executors.newCachedThreadPool();
private final static long timeMillis = System.currentTimeMillis();
static String FileName="";
public static void main(String[] args) throws MalformedURLException {
int i=0;
ArrayList<String> list = new ArrayList<>();
String str="";
Scanner scanner = new Scanner(System.in);
System.out.println("请输入想要搜索的内容");
str=scanner.next();
Document connect = null;
try {
connect = Jsoup.connect("https://search.bilibili.com/all?keyword="+str).get();
} catch (IOException e) {
throw new RuntimeException(e);
}
Elements li = connect.getElementsByTag("li");
for (Element element : li) {
String url = element.getElementsByTag("a").attr("href");
if (url.contains("www.bilibili.com/video/BV")){
list.add(url.substring(url.indexOf("BV")+2,url.indexOf("?")));
}
}
long start = System.currentTimeMillis();
DownloadVivw(list,i);
long end = System.currentTimeMillis();
//System.err.println("一共下载:"+list.size()+"份视频,总共耗时:" + (end - start) / 1000 + "s");
}
public static void DownloadVivw(final ArrayList<String> list, int i){
if (i==list.size()){
return;
}else {
//通过bv号自动下载视频
String bvid = list.get(i);
//建立连接,先获取到 cid
String cidJson = getCid(bvid);
//获取 视频真实路径
String url = getVivw(cidJson,bvid);
System.out.println("开始");
long start = System.currentTimeMillis();
downloadMovie(url,FileName);
long end = System.currentTimeMillis();
System.out.println("以下载"+(i+1)+"份视频");
System.err.println("总共耗时:" + (end - start) / 1000 + "s,线程名"+Thread.currentThread().getName());
i++;
final int finalI = i;
DownloadVivw(list, finalI);
/* cacheExecutor.execute(new Runnable() {
@Override
public void run() {
DownloadVivw(list, finalI);
}
});*/
}
long end = System.currentTimeMillis();
System.err.println("总共耗时:" + (end - timeMillis) / 1000 + "s");
}
private static String getVivw(String cidJson,String bvid) {
//qn : 视频质量 112 -> 高清 1080P+, 80 -> 高清 1080P, 64 -> 高清 720P, 32 -> 清晰 480P, 16 -> 流畅 360P
// 最高支持 1080p, 1080P+是不支持的
Integer qn = 80;
String paraUrl = "https://api.bilibili.com/x/player/playurl?cid=" + cidJson + "&fnver=0&qn=" + qn + "&otype=json&bvid=" + bvid + "&fnval=2&player=1";
StringBuilder json = new StringBuilder();
try {
URL urlObject = new URL(paraUrl);
URLConnection urlConnection = urlObject.openConnection();
BufferedReader in = new BufferedReader(new InputStreamReader(urlConnection.getInputStream(),"utf-8"));
String inputLine = null;
while ( (inputLine = in.readLine()) != null) {
json.append(inputLine);
}
in.close();
} catch (Exception e) {
e.printStackTrace();
}
json.toString();
Map maps = (Map) JSON.parse(String.valueOf(json));
Object o = ((Map) maps.get("data")).get("durl");
return (String)((JSONObject) ((JSONArray) o).get(0)).get("url");
}
private static String getCid(String avid) {
String CidUrl="https://api.bilibili.com/x/web-interface/view?bvid="+avid;
StringBuilder json = new StringBuilder();
try {
URL urlObject = new URL(CidUrl);
URLConnection urlConnection = urlObject.openConnection();
BufferedReader in = new BufferedReader(new InputStreamReader(urlConnection.getInputStream(),"utf-8"));
String inputLine = null;
while ( (inputLine = in.readLine()) != null) {
json.append(inputLine);
}
in.close();
} catch (Exception e) {
e.printStackTrace();
}
json.toString();
Map maps = (Map) JSON.parse(String.valueOf(json));
FileName=((Map)maps.get("data")).get("title")+".flv";
return ((Map)maps.get("data")).get("cid")+"";
}
public static void downloadMovie(String BLUrl, String fileName) {
InputStream inputStream = null;
try {
URL url = new URL(BLUrl);
URLConnection urlConnection = url.openConnection();
urlConnection.setRequestProperty("Referer", "https://www.bilibili.com/video/BV14S4y127Gd"); // 填需要爬取的bv号
urlConnection.setRequestProperty("Sec-Fetch-Mode", "no-cors");
urlConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36");
urlConnection.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
urlConnection.setConnectTimeout(10 * 1000);
inputStream = urlConnection.getInputStream();
} catch (IOException e) {
e.printStackTrace();
}
//定义路径
String path = "C:\\file\\img\\" + fileName;
File file = new File(path);
int i = 1;
try {
BufferedInputStream bis = new BufferedInputStream(inputStream);
BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file));
byte[] bys = new byte[1024];
int len = 0;
while ((len = bis.read(bys)) != -1) {
bos.write(bys, 0, len);
}
bis.close();
bos.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}