多线程下载m3u8影视资源 通过ffmpeg合并ts文件为mp4

需要用到的ffmpeg，下载地址：Download FFmpeg 

下载以后的目录。

 

 合并、转换、切片都用到了ffmpeg.exe，其它两个我暂时没用到。

<dependency>
         <groupId>com.squareup.okhttp3</groupId>
         <artifactId>okhttp</artifactId>
         <version>3.14.2</version>
 </dependency>
<dependency>
	    <groupId>org.jsoup</groupId>
	    <artifactId>jsoup</artifactId>
	    <version>1.13.1</version>
	</dependency>

java里面用到了：Jsoup、OKHTTP

 以某影视资源提供的m3u8为例：https://vod1.XXX.com/20220424/w9SgsDXT/index.m3u8

package cc.gaole.video.server;

import java.io.File;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.util.Calendar;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadPoolExecutor;

import org.apache.commons.collections.CollectionUtils;

import com.gaole.video.DateUtil;
import com.gaole.video.HttpUtils;
import com.gaole.video.resources.ResourcesTemplate;

import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;

/**
*抽象类(因为我实现了几种下载视频的方式，所以定义了一个抽象类，把公共实现的方法都独立出来)
*
**/
public abstract class AbstractAppServer implements Callable<AbstractAppServer>{
	protected String ffmpegPath= null;
	protected String targetFileName = null; //下载的视频定义的名字
	protected String url = null; //下载视频的URL
	protected String rootDir ="F:\\temp"; //下载的ts文件存放目录
	protected String movieDir = "F:\\movie";//ts文件处理完成并且合并成mp4后需要移动的目录
	protected String tempHandleDir = null; //多部电影下载，则每个处理视频，都单独生成一个目录
	protected int corePoolSize = 20;
	protected int queueSize = 3000;
	protected ThreadPoolExecutor excuterManagerPool =null;
	// 单次计数器，用于计算子类里面的线程是否全部处理完了
	protected  CountDownLatch countDownLatch = null;
	/**
    *获取index.m3u8返回的所有ts文件
    */
	protected  List<String> getVideoUrlText(){
    	List<String> list = null;
		try {
			list = ResourcesTemplate.getSuperVideoUrlText(url);
		} catch (Exception e) {
			e.printStackTrace();
			return list;
		}
		
		return list;
    }
	
	protected void downTsFile(List<String> tsList,String rootDir){
    	if(CollectionUtils.isEmpty(tsList)){return;}
    	
    	try {
    		OkHttpClient client= HttpUtils.getHttpClient(true);
        	File dirFile = new File(rootDir);
        	if(!dirFile.exists()){
        		dirFile.mkdirs();
        	}
        	int fileSize = tsList.size();
        	countDownLatch = new CountDownLatch(fileSize);
        	for(int i=0;i<fileSize;i++){
        		String str = tsList.get(i);
        		int tempI = i;
        		excuterManagerPool.submit(new Thread(()->{
        			boolean dowFlag = this.downSignTsFile(client,str,dirFile,1,tempI);
        			countDownLatch.countDown();
            	}));
        	}
			countDownLatch.await();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
    }
    
    private boolean downSignTsFile(OkHttpClient client,String str,File dirFile,int downNum,int downFileName){
    	Request request = new Request.Builder().url(str).get().addHeader("Connection", "keep-alive").build();
    	Response response = null;
		try {
			response = client.newCall(request).execute();
			int responseCode = response.code();
    		if(responseCode == 200){
    			
    			//String fileName = str.substring(str.lastIndexOf("/"),str.lastIndexOf("."));
    			String suffix = str.substring(str.lastIndexOf(".")+1);
    			String newName = dirFile.getAbsolutePath()+"\\"+downFileName+'.'+suffix;
    			byte[] bt = response.body().bytes();
    			File localFile = new File(newName);
    			if(localFile.exists()){
    				localFile.delete();
    			}else{
    				localFile.createNewFile();
    			}
    			
    			Files.write(localFile.toPath(), bt, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
    			return true;
    		}
    		
		} catch (Exception e) {
			if(downNum <=3){
				System.out.println(targetFileName+"第 "+(downFileName+1)+"个文件下载失败:"+e.getMessage()+" ，3秒后重新下载");
				try {
					Thread.currentThread().sleep(3000);
					boolean resultFlag = downSignTsFile(HttpUtils.getHttpClient(true),str,dirFile,downNum+1,downFileName);
					return resultFlag;
				} catch (InterruptedException e1) {
				}
			}else{
				e.printStackTrace();
				
			}
		}finally{
			if(response !=null){
				response.close();
			}
		}
		return false;
    }
    
    /**
     * 删除目录下的文件
     * @param tempHandleDir
     * @param deleteFolder 是否删除目录下的文件夹
     */
    protected boolean deleTempFile(String tempHandleDir,boolean deleteFolder){
    	File tempFile = new File(tempHandleDir);
    	
    	File[] files = tempFile.listFiles();
    	if(files == null){return true;}
    	
    	int fileSize = files.length;
    	if(fileSize <=0){return true;}
    	
    	for(int i=0;i<fileSize;i++){
    		File file = files[i];
    		if(file.isDirectory() && deleteFolder){
    			deleTempFile(file.getPath(),deleteFolder);
    			file.delete();
    		}else if(file.isFile()){
    			file.delete();
    		}
    	}
    	
    	return true;
    }
	
    protected  String createNewFolderName(){
    	return DateUtil.parseStr(Calendar.getInstance().getTime(), DateUtil.FMT_YYYYMMDDHHMM);
    }
    
    protected void fileMoveTarget(File oldFile,String targetMovieDir){
    	File targetDir = new File(targetMovieDir);
    	if(!targetDir.exists()){
    		targetDir.mkdirs();
    	}
    	
    	String moveFileName = oldFile.getName();
    	File targetFile = new File(targetMovieDir+"\\"+moveFileName);
    	if(targetFile.exists()){targetFile.delete();}
    	
    	oldFile.renameTo(targetFile);
   }
}

package cc.gaole.video;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import org.apache.commons.collections.CollectionUtils;

import cc.gaole.video.server.AbstractAppServer;

/**
 * Hello world!
 *
 */
public class AppServer extends AbstractAppServer{
	private AppServer(){}
	
	public AppServer(String url,String targetFileName,String ffmpegPath){
		this.url = url.toString();
		this.targetFileName=targetFileName;
		this.ffmpegPath=ffmpegPath;
		excuterManagerPool =new ThreadPoolExecutor(corePoolSize, corePoolSize, 0L, TimeUnit.MILLISECONDS,new LinkedBlockingQueue<Runnable>(queueSize));
		tempHandleDir = rootDir+"\\"+this.createNewFolderName()+(new Random().nextInt()*1000);
	}
	
	@Override
	public AppServer call() throws Exception {
		System.out.println(targetFileName+"开始时间："+DateUtil.parseStr(new Date(), DateUtil.FMT_YYYYMMDD_HHMMSS));
		this.deleTempFile(tempHandleDir, true);
		if(m3u8Flag){
			downM3U8(tempHandleDir);
		}else{
			downVideo(tempHandleDir);
		}
		this.deleTempFile(tempHandleDir, true);
    	//把生成的根目录删掉
    	new File(tempHandleDir).delete();
		System.out.println(targetFileName+"结束时间："+DateUtil.parseStr(new Date(), DateUtil.FMT_YYYYMMDD_HHMMSS));
		return this;
	}
	
	private void downM3U8(String tempHandleDir){
    	List<String> tsList = this.getVideoUrlText();
    	if(CollectionUtils.isEmpty(tsList) || tsList.size() <=0){System.out.println(targetFileName+"未获取到片源");return;}
    	
    	this.downTsFile(tsList,tempHandleDir);
    	//合并、移动文件
    	this.mergeTs(tempHandleDir);
	}
    
    //合并文件
    private void mergeTs(String tsDir){
    	File tsDirFile = new File(tsDir);
    	File[] files = tsDirFile.listFiles();
    	int fileSize = files.length;
    	if(fileSize <=0){return;}
    	
    	Arrays.sort(files, new Comparator<File>() {

			@Override
			public int compare(File f1, File f2) {
				int f1Num =Integer.parseInt((f1.getName().substring(0, f1.getName().indexOf("."))));
				int f2Num =Integer.parseInt((f2.getName().substring(0, f2.getName().indexOf("."))));
				long diff = f1Num - f2Num; //f1.lastModified() - f2.lastModified();
                if (diff > 0)
                    return 1;
                else if (diff == 0)
                    return 0;
                else
                    return -1;//如果 if 中修改为 返回-1 同时此处修改为返回 1  排序就会是递减,如果 if 中修改为 返回1 同时此处修改为返回 -1  排序就会是递增,
			}
    		
    	});
    	
    	//创建一个完成目录
    	File succesDir = new File(tsDir+"\\完成");
    	if(succesDir.exists()){
    		succesDir.delete();
    	}
    	succesDir.mkdir();
    	
    	File endFile = new File(succesDir+"\\"+targetFileName+".ts");
    	if (endFile.exists())
    		endFile.delete();
        else {
        	try {
        		endFile.createNewFile();
			} catch (Exception e) {
			}
        }
    	
    	FileOutputStream fileOutputStream = null;
    	try {
    		fileOutputStream = new FileOutputStream(endFile);
    		byte[] b = new byte[4096];
        	
        	for(int i=0;i<fileSize;i++){
        		File file = files[i];
        		FileInputStream fileInputStream = new FileInputStream(file);
        		 
        		int len;
                while ((len = fileInputStream.read(b)) != -1) {
                    fileOutputStream.write(b, 0, len);
                }
                fileInputStream.close();
                fileOutputStream.flush();
                
        	}
        	fileOutputStream.close();
		} catch (Exception e) {
			e.printStackTrace();
			if(fileOutputStream !=null){
				try {
					fileOutputStream.close();
				} catch (IOException e1) {
				}
			}
		}
    	
    	//视频转码成mp4
    	File convertVideoFile = this.fileConvertVideo(endFile);
    	//文件移动
    	if(convertVideoFile !=null && convertVideoFile.exists()){
    		this.fileMoveTarget(convertVideoFile, movieDir);
    	}else{
    		System.out.println(targetFileName+"格式转换失败");
    	}
    }
    
    private File fileConvertVideo(File oldFile){
    	 Runtime runtime = null;
    	 File mp4File = new File(oldFile.getParent()+"\\"+(oldFile.getName().substring(0,oldFile.getName().indexOf(".")))+".mp4");
    	try {
    		StringBuilder sb = new StringBuilder(this.ffmpegPath);
        	sb.append(" -y");
        	sb.append(" -i");
        	sb.append(" "+oldFile.getAbsolutePath());
        	sb.append(" -threads 2");
        	sb.append("  -c:v copy -c:a copy");
        	sb.append(" "+mp4File.getAbsolutePath());
        	runtime = Runtime.getRuntime();
			Process process =runtime.exec(sb.toString());
			BufferedReader br = new BufferedReader(new InputStreamReader(process.getErrorStream()));
            String line = null;
            while((line = br.readLine()) != null){
            	//System.out.println("<<视频执行信息>> "+line);
            }
            br.close();
            process.getOutputStream().close();
            process.getInputStream().close();
            process.getErrorStream().close();
		} catch (IOException e) {
			e.printStackTrace();
			return null;
		} finally{
		}
    	return mp4File;
    }
    
}

main方法调用

package cc.gaole.video;

import java.io.File;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import cc.gaole.video.server.AbstractAppServer;

 * 2022年6月20日
 */
public class VideoCoreApp {
	private static final String ffmpegPath = "F:\\ffmpeg-4.4.1-essentials_build\\ffmpeg-4.4.1-essentials_build\\bin\\ffmpeg.exe"; 
	private ThreadPoolExecutor excuterManagerPool =null;
	public VideoCoreApp(){
		excuterManagerPool =new ThreadPoolExecutor(5, 5, 0L, TimeUnit.MILLISECONDS,new LinkedBlockingQueue<Runnable>(100));
	}
	
	public void addTask(AbstractAppServer appServer){
		this.excuterManagerPool.submit(appServer);
	}
	
	public static void main(String[] args) {
		
		try {
			
   			String ffmpegPath = "F:\\ffmpeg-4.4.1-essentials_build\\ffmpeg-4.4.1-essentials_build\\bin\\ffmpeg.exe"; 
   			VideoCoreApp core = new VideoCoreApp();
   			core.addTask(new AppServer("https://vod1.xxx.com/20220424/w9SgsDXT/index.m3u8", "仙武帝尊第一集",ffmpegPath));
   			
		
   		} catch (Exception e) {
   			e.printStackTrace();
   		}
		
	}
}

里面掺杂了很多自己的东西，所以不想单独分解出来了，主要为了自己做个笔记记录起来，方便以后可以参阅。

main方法里面定义了一个线程池，可以一次性下载几部资源，在子类里面AppServer也定义了一个线程池可以一次性处理多少个ts文件。

执行流程：

1、VideoCoreApp 的 main方法 初始化主线程

2、AppServer 的构造方法 初始化子线程

3、 AppServer 的call()   --- 主方法

4、AppServer  的downM3U8()  --- 主方法的具体实现方法

5、AbstractAppServer 的 getVideoUrlText() --- 获取index.m3u8需要下载的ts文件  和 downTsFile() --- 下载ts文件

6、AppServer 的 mergeTs()   --- 将下载好的ts文件合并成mp4文件，并且移动到一个完成的目录

需要用到的工具类

package cc.gaole.video.resources;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
//获取资源网站ts文件
public abstract class ResourcesTemplate {

	public String parseUrl = null;
	public String url = "";
	public abstract  List<String> getVideoUrlText(String url) throws Exception;
	
	public static List<String> getSuperVideoUrlText(String url) throws Exception{
        //有些资源网站比较特殊，要单独处理，这里只给大家提供通用的即可
		ResourcesTemplate resources =  new CommonResources(url);

		return resources.getVideoUrlText(url);
	}
	
	public  List<String> getTsUrlList(String resultText,String absolutePrefixPath) {
		List<String> list = new ArrayList<String>();
		String [] strs = resultText.split(",");
		for(String urlStr :strs){
			if(!urlStr.contains(".ts")){continue;}
			Pattern pattern = Pattern.compile("([http|https]?.*\\.ts)");
			Matcher m = pattern.matcher(urlStr);
			if (m.find()) {
				String urlTemp = m.group(0);
				if(urlTemp.charAt(0) != '/'){
					urlTemp = "/"+urlTemp;
				}
				list.add(urlTemp.startsWith("http") ? urlTemp : ((StringUtils.isEmpty(absolutePrefixPath) ? parseUrl : absolutePrefixPath)+urlTemp));
			}
		}
		return list;
	}
}

package cc.gaole.video.resources;

import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;

import cc.gaole.video.FileUtil;
import cc.gaole.video.HttpUtils;
import cc.gaole.video.SSLSocketClient;

import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;

public class CommonResources extends ResourcesTemplate{
	
	public CommonResources(String url){
		this.url = url;
	}

	@Override
	public List<String> getVideoUrlText(String url) throws Exception {
		List<String> list = null;
		OkHttpClient client= HttpUtils.getHttpClient(true);
		Response response = null;
		try {
			Request request = new Request.Builder().url(url).get().addHeader("Connection", "keep-alive").build();
			response = client.newCall(request).execute();
			int responseCode = response.code();
			if(responseCode == 200){
				String urlText  = response.body().string();
				if(urlText.contains("EXTM3U") && urlText.length() >20){
					URL host = new URL(url);
					if(urlText.contains(".m3u8")){
						//有的资源要进行第二次获取ts的文件
						Pattern pattern = Pattern.compile("([a-z|A-Z|/|0-9]+.*.m3u8)");
						Matcher m = pattern.matcher(urlText);
						if (m.find()) {
							urlText  = m.group(0); 
							if(urlText.charAt(0) != '/'){
								urlText = "/"+urlText;
							}						
							if(!urlText.contains("http")){
								urlText = HttpUtils.getHostAbsolutePath(host)+ urlText;
							}
							
							SSLSocketClient.uncheckSSL();
							urlText = Jsoup.connect(urlText).ignoreContentType(true).execute().body();  
						}
					}
					
					list = super.getTsUrlList(urlText, HttpUtils.getHostAbsolutePath(host));
				}
			}else{
				System.out.println(url+"资源访问返回状态："+responseCode);
			}
		} catch (Exception e) {
			throw new Exception(e);
		} finally{
			if(response !=null){
				response.close();
			}
		}
		
		return list;
	}

}

package cc.gaole.video;

import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.TimeUnit;

import okhttp3.ConnectionPool;
import okhttp3.Cookie;
import okhttp3.CookieJar;
import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
import okhttp3.OkHttpClient.Builder;

public class HttpUtils {
	
	private static final int connectTimeout= 10000;
	private static final int readTimeout = 10000;
	private static final int writeTimeout=10000;
	private static ConnectionPool mConnectionPool=new ConnectionPool(1000, 30, TimeUnit.MINUTES);
	
	public static OkHttpClient getHttpClient(boolean ignoreSSLChecked){
		Builder builder = new OkHttpClient.Builder().cookieJar(new CookieJar() {
			private final HashMap<String, List<Cookie>> cookieStore = new HashMap<String, List<Cookie>>();
			@Override
			public void saveFromResponse(HttpUrl url, List<Cookie> cookies) {
				cookieStore.put(url.host(), cookies);
				
			}
			
			@Override
			public List<Cookie> loadForRequest(HttpUrl url) {
				List<Cookie> cookies = cookieStore.get(url.host());
		        return cookies != null ? cookies : new ArrayList<Cookie>();
			}
		})
		.followRedirects(false)
		.followSslRedirects(false)
		.connectTimeout(connectTimeout, TimeUnit.MILLISECONDS) //连接超时
        .readTimeout(readTimeout, TimeUnit.MILLISECONDS) //读取超时
        .writeTimeout(writeTimeout, TimeUnit.MILLISECONDS) //写超时
        .connectionPool(mConnectionPool);
		if(ignoreSSLChecked){
			  //忽略SSL验证
			builder.sslSocketFactory(SSLSocketClient.getSSLSocketFactory(),SSLSocketClient.getX509TrustManager())
	    		   .hostnameVerifier(SSLSocketClient.getHostnameVerifier());
		}
		
		OkHttpClient httpClient =builder.build();
		return httpClient;
	}
	
	public static String getHostAbsolutePath(URL host){
		Integer port  = host.getPort();
		String urlPath = host.getProtocol()+"://"+ host.getHost()+(port !=null && port.intValue() >=1 ? ":"+port : ""); 
		return urlPath;
	}
}

package cc.gaole.video;

import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Arrays;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
/**
 *  不想再本地使用证书访问网站，可以忽略SSL验证
 */
public class SSLSocketClient {
	
	/**
	 * 忽略SSL验证，需配合jsonup 或者 httpConnection
	 */
	static public void uncheckSSL() {
        try {
            SSLContext context = SSLContext.getInstance("TLS");
            context.init(null, new X509TrustManager[]{new X509TrustManager() {
                public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                }

                public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                }

                public X509Certificate[] getAcceptedIssuers() {
                    return new X509Certificate[0];
                }
            }}, new SecureRandom());
            HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());
        } catch (NoSuchAlgorithmException e) {
        } catch (KeyManagementException e) {
        }
    }

	//获取这个SSLSocketFactory
    public static SSLSocketFactory getSSLSocketFactory() {
        try {
            SSLContext sslContext = SSLContext.getInstance("SSL");
            sslContext.init(null, getTrustManager(), new SecureRandom());
            return sslContext.getSocketFactory();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    
  //获取TrustManager
    private static TrustManager[] getTrustManager() {
        return new TrustManager[]{
                new X509TrustManager() {
                    @Override
                    public void checkClientTrusted(X509Certificate[] chain, String authType) {
                    }
 
                    @Override
                    public void checkServerTrusted(X509Certificate[] chain, String authType) {
                    }
 
                    @Override
                    public X509Certificate[] getAcceptedIssuers() {
                        return new X509Certificate[]{};
                    }
                }
        };
    }
 
    //获取HostnameVerifier
    public static HostnameVerifier getHostnameVerifier() {
        return (s, sslSession) -> true;
    }
    
    public static X509TrustManager getX509TrustManager() {
        X509TrustManager trustManager = null;
        try {
            TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
            trustManagerFactory.init((KeyStore) null);
            TrustManager[] trustManagers = trustManagerFactory.getTrustManagers();
            if (trustManagers.length != 1 || !(trustManagers[0] instanceof X509TrustManager)) {
                throw new IllegalStateException("Unexpected default trust managers:" + Arrays.toString(trustManagers));
            }
            trustManager = (X509TrustManager) trustManagers[0];
        } catch (Exception e) {
            e.printStackTrace();
        }
 
        return trustManager;
    }
}