Javacv 视频流转换 hsl、rtsp、rtmp 转 flv

      最近接触了视频流相关转换内容，针对视频流转换做一下记录，主要记录后端代码部分。前端使用flv.js + WebSocket 实现视频流播放。

服务依赖准备

        <dependency>
            <groupId>org.bytedeco</groupId>
            <artifactId>javacv</artifactId>
            <version>1.5.9</version>
        </dependency>
        <dependency>
            <groupId>org.bytedeco</groupId>
            <artifactId>opencv-platform-gpu</artifactId>
            <version>4.7.0-1.5.9</version>
        </dependency>
        <!-- Optional GPL builds with (almost) everything enabled -->
        <dependency>
            <groupId>org.bytedeco</groupId>
            <artifactId>ffmpeg</artifactId>
            <version>6.0-1.5.9</version>
            <classifier>linux-x86_64</classifier>
        </dependency>
        <dependency>
            <groupId>org.bytedeco</groupId>
            <artifactId>ffmpeg</artifactId>
            <version>6.0-1.5.9</version>
            <classifier>windows-x86_64</classifier>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.69</version>
        </dependency>

Service实现

service

public interface IFLVService {

	/**
	 * 打开一个流地址
	 * 
	 * @param url 视频流地址
	 * @param response
	 * @param request
	 */
	public void open(String url, HttpServletResponse response, HttpServletRequest request) throws Exception;

	/**
	 * 关闭视频流
	 * @param url
	 */
	public void close(String url);
}

impl

/**
 * FLV流转换
 * 
 * @author gc.x
 */
@Slf4j
@Service
public class FLVService implements IFLVService {

	private ExecutorService executor;

	private ConcurrentHashMap<String, Converter> converters = new ConcurrentHashMap<>();

	private ConcurrentHashMap<String, Future<?>> taskMap = new ConcurrentHashMap<>();

	@PostConstruct
	public void initExecutor() {
		executor = ThingsBoardExecutors.newWorkStealingPool(50, "video-executor");
	}

	@PreDestroy
	public void shutdownExecutor() {
		if (executor != null) {
			executor.shutdownNow();
		}
	}

	@Override
	public void open(String url, HttpServletResponse response, HttpServletRequest request) throws Exception {
		String key = md5(url);
		AsyncContext async = request.startAsync();
		async.setTimeout(0);
		if (converters.containsKey(key)) {
			Converter c = converters.get(key);
		} else {
			List<AsyncContext> outs = Lists.newArrayList();
			outs.add(async);
			ConverterFactories c = new ConverterFactories(url, key, converters, outs);
			c.start();
			converters.put(key, c);
		}
	}

	public String md5(String plainText) {
		StringBuilder buf = null;
		try {
			MessageDigest md = MessageDigest.getInstance("MD5");
			md.update(plainText.getBytes());
			byte b[] = md.digest();
			int i;
			buf = new StringBuilder("");
			for (int offset = 0; offset < b.length; offset++) {
				i = b[offset];
				if (i < 0)
					i += 256;
				if (i < 16)
					buf.append("0");
				buf.append(Integer.toHexString(i));
			}
		} catch (NoSuchAlgorithmException e) {
			log.error(e.getMessage(), e);
		}
		return buf.toString();
	}


	@Override
	public void close(String url){
		String key = md5(url);
		if (converters.containsKey(key)) {
			Converter c = converters.get(key);
			c.exit();
			converters.remove(key);
		}
	}
}

转换

当前使用的输出流为WebSocket，通过WebSocket推送字节流。

Convert

public interface Converter {

	/**
	 * 获取该转换的key
	 */
	public String getKey();

	/**
	 * 获取该转换的url
	 * 
	 * @return
	 */
	public String getUrl();

	/**
	 * 添加一个流输出
	 * 
	 * @param entity
	 */
	public void addOutputStreamEntity(String key, AsyncContext entity) throws IOException;

	/**
	 * 退出转换
	 */
	public void exit();

	/**
	 * 启动
	 */
	public void start();

}

实现 ConverterFactories

/**
 * javacv转包装<br/>
 * 无须转码，更低的资源消耗，更低的延迟<br/>
 * 确保流来源视频H264格式,音频AAC格式
 * 
 * @author gc.x
 */
@Slf4j
public class ConverterFactories extends Thread implements Converter {
	public volatile boolean runing = true;
	/**
	 * 读流器
	 */
	private FFmpegFrameGrabber grabber;
	/**
	 * 转码器
	 */
	private FFmpegFrameRecorder recorder;
	/**
	 * 转FLV格式的头信息<br/>
	 * 如果有第二个客户端播放首先要返回头信息
	 */
	private byte[] headers;
	/**
	 * 保存转换好的流
	 */
	private ByteArrayOutputStream stream;
	/**
	 * 流地址，h264,aac
	 */
	private String url;
	/**
	 * 流输出
	 */
	private List<AsyncContext> outEntitys;

	/**
	 * key用于表示这个转换器
	 */
	private String key;

	/**
	 * WebSocket推送对象
	 */
	private SessionMetaData metaData;

	/**
	 * 转换队列
	 */
	private Map<String, Converter> factories;

	public ConverterFactories(String url, String key, Map<String, Converter> factories, List<AsyncContext> outEntitys) {
		this.url = url;
		this.key = key;
		this.factories = factories;
		this.outEntitys = outEntitys;
	}

	@Override
	public void run() {
		boolean isCloseGrabberAndResponse = true;
		try {
			grabber = new FFmpegFrameGrabber(url);
			if ("rtsp".equals(url.substring(0, 4))) {
				grabber.setOption("rtsp_transport", "tcp");
				// 首选TCP进行RTP传输
				grabber.setOption("rtsp_flags", "prefer_tcp");
			} else if ("rtmp".equals(url.substring(0, 4))) {
				// rtmp拉流缓冲区，默认3000毫秒
				grabber.setOption("rtmp_buffer", "1000");
				// 默认rtmp流为直播模式，不允许seek
				// grabber.setOption("rtmp_live", "live");

			}
			grabber.start();
			if (avcodec.AV_CODEC_ID_H264 == grabber.getVideoCodec()
					&& (grabber.getAudioChannels() == 0 || avcodec.AV_CODEC_ID_AAC == grabber.getAudioCodec())) {
				log.info("this url:{} converterFactories start", url);
				// 来源视频H264格式,音频AAC格式
				// 无须转码，更低的资源消耗，更低的延迟
				stream = new ByteArrayOutputStream();
				recorder = new FFmpegFrameRecorder(stream, grabber.getImageWidth(), grabber.getImageHeight(),
						grabber.getAudioChannels());
				recorder.setInterleaved(true);
				recorder.setVideoOption("preset", "ultrafast");
				recorder.setVideoOption("tune", "zerolatency");
				recorder.setVideoOption("crf", "25");
				recorder.setFrameRate(grabber.getFrameRate());
				recorder.setSampleRate(grabber.getSampleRate());
				if (grabber.getAudioChannels() > 0) {
					recorder.setAudioChannels(grabber.getAudioChannels());
					recorder.setAudioBitrate(grabber.getAudioBitrate());
					recorder.setAudioCodec(grabber.getAudioCodec());
				}
				recorder.setFormat("flv");
				recorder.setVideoBitrate(grabber.getVideoBitrate());
				recorder.setVideoCodec(grabber.getVideoCodec());
				recorder.start(grabber.getFormatContext());

				int nullNumber = 0;
				while (runing) {
					if (metaData != null) {
						if (headers == null) {
							headers = stream.toByteArray();
							stream.reset();
							writeResponse(headers);
						}
						// 抓取一帧
						Frame f = grabber.grab();
						if (f != null) {
							try {
								// 转码
								recorder.record(f);
							} catch (Exception e) {
							}
							if (stream.size() > 0) {
								byte[] b = stream.toByteArray();
								stream.reset();
								writeResponse(b);
							}
						} else {
							nullNumber++;
							if (nullNumber > 200) {
								break;
							}
						}
						Thread.sleep(5);
					}
				}
			} else {
				isCloseGrabberAndResponse = false;
				// 需要转码为视频H264格式,音频AAC格式
				ConverterTranFactories c = new ConverterTranFactories(url, key, factories, outEntitys, grabber, metaData);
				factories.put(key, c);
				c.start();
			}
		} catch (Exception e) {
			log.error(e.getMessage(), e);
		} finally {
			closeConverter(isCloseGrabberAndResponse);
			completeResponse(isCloseGrabberAndResponse);
			log.info("this url:{} converterFactories exit", url);

		}
	}

	/**
	 * 输出FLV视频流
	 *
	 * @param b
	 */
	public void writeResponse(byte[] b) {
		if (this.metaData != null) {
			try {
				data.basicRemote.sendBinary(ByteBuffer.wrap(b));
			} catch (IOException e) {
				log.error("Sending video data error, msg : [{}]", e.getMessage());
			}
		}
	}

	/**
	 * 退出转换
	 */
	public void closeConverter(boolean isCloseGrabberAndResponse) {
		if (isCloseGrabberAndResponse) {
			IOUtils.close(grabber);
			factories.remove(this.key);
		}
		IOUtils.close(recorder);
		IOUtils.close(stream);
	}

	/**
	 * 关闭异步响应
	 *
	 * @param isCloseGrabberAndResponse
	 */
	public void completeResponse(boolean isCloseGrabberAndResponse) {
		if (isCloseGrabberAndResponse) {
			Iterator<AsyncContext> it = outEntitys.iterator();
			while (it.hasNext()) {
				AsyncContext o = it.next();
				o.complete();
			}
		}
	}

	@Override
	public String getKey() {
		return this.key;
	}

	@Override
	public String getUrl() {
		return this.url;
	}

	@Override
	public void addOutputStreamEntity(String key, AsyncContext entity) throws IOException {
		if (headers == null) {
			outEntitys.add(entity);
		} else {
			entity.getResponse().getOutputStream().write(headers);
			entity.getResponse().getOutputStream().flush();
			outEntitys.add(entity);
		}
	}

	@Override
	public void exit() {
		this.runing = false;
		try {
			this.join();
		} catch (Exception e) {
			log.error(e.getMessage(), e);
		}
	}

}

实现 ConverterTranFactories

/**
 * javacv转码<br/>
 * 流来源不是视频H264格式,音频AAC格式 转码为视频H264格式,音频AAC格式
 *
 * @author gc.x
 */
@Slf4j
public class ConverterTranFactories extends Thread implements Converter {
	public volatile boolean runing = true;
	/**
	 * 读流器
	 */
	private FFmpegFrameGrabber grabber;
	/**
	 * 转码器
	 */
	private FFmpegFrameRecorder recorder;
	/**
	 * 转FLV格式的头信息<br/>
	 * 如果有第二个客户端播放首先要返回头信息
	 */
	private byte[] headers;
	/**
	 * 保存转换好的流
	 */
	private ByteArrayOutputStream stream;
	/**
	 * 流地址，h264,aac
	 */
	private String url;
	/**
	 * 流输出
	 */
	private List<AsyncContext> outEntitys;

	/**
	 * key用于表示这个转换器
	 */
	private String key;

	/**
	 * 转换队列
	 */
	private Map<String, Converter> factories;

	/**
	 * WebSocket推送对象
	 */
	private SessionMetaData metaData;

	public ConverterTranFactories(String url, String key, Map<String, Converter> factories,
								  List<AsyncContext> outEntitys, FFmpegFrameGrabber grabber,
								  SessionMetaData metaData) {
		this.url = url;
		this.key = key;
		this.factories = factories;
		this.outEntitys = outEntitys;
		this.grabber = grabber;
		this.metaData = metaData;
	}

	@Override
	public void run() {
		try {
			log.info("this url:{} converterTranFactories start", url);
			grabber.setFrameRate(25);
			if (grabber.getImageWidth() > 1920) {
				grabber.setImageWidth(1920);
			}
			if (grabber.getImageHeight() > 1080) {
				grabber.setImageHeight(1080);
			}
			stream = new ByteArrayOutputStream();
			recorder = new FFmpegFrameRecorder(stream, grabber.getImageWidth(), grabber.getImageHeight(),
					grabber.getAudioChannels());
			recorder.setInterleaved(true);
			recorder.setVideoOption("preset", "ultrafast");
			recorder.setVideoOption("tune", "zerolatency");
			recorder.setVideoOption("crf", "25");
			recorder.setGopSize(50);
			recorder.setFrameRate(25);
			recorder.setSampleRate(grabber.getSampleRate());
			if (grabber.getAudioChannels() > 0) {
				recorder.setAudioChannels(grabber.getAudioChannels());
				recorder.setAudioBitrate(grabber.getAudioBitrate());
				recorder.setAudioCodec(avcodec.AV_CODEC_ID_AAC);
			}
			recorder.setFormat("flv");
			recorder.setVideoBitrate(grabber.getVideoBitrate());
			recorder.setVideoCodec(avcodec.AV_CODEC_ID_H264);
			recorder.start();

			int nullNumber = 0;
			while (runing) {
				if (metaData != null) {
					if (headers == null) {
						headers = stream.toByteArray();
						stream.reset();
						writeResponse(headers);
					}
					// 抓取一帧
					Frame f = grabber.grab();
					if (f != null) {
						try {
							// 转码
							recorder.record(f);
						} catch (Exception e) {
						}
						if (stream.size() > 0) {
							byte[] b = stream.toByteArray();
							stream.reset();
							writeResponse(b);
						}
					} else {
						nullNumber++;
						if (nullNumber > 200) {
							break;
						}
					}
					Thread.sleep(5);
				}
			}
		} catch (Exception e) {
			log.error(e.getMessage(), e);
		} finally {
			closeConverter();
			completeResponse();
			log.info("this url:{} converterTranFactories exit", url);
			factories.remove(this.key);
		}
	}

	/**
	 * 输出FLV视频流
	 *
	 * @param b
	 */
	public void writeResponse(byte[] b) {
		if (this.metaData != null) {
			try {
				metaData .basicRemote.sendBinary(ByteBuffer.wrap(b));
			} catch (IOException e) {
				log.error("Sending video data error, msg : [{}]", e.getMessage());
			}
		}
	}

	/**
	 * 退出转换
	 */
	public void closeConverter() {
		IOUtils.close(grabber);
		IOUtils.close(recorder);
		IOUtils.close(stream);
	}

	/**
	 * 关闭异步响应
	 */
	public void completeResponse() {
		Iterator<AsyncContext> it = outEntitys.iterator();
		while (it.hasNext()) {
			AsyncContext o = it.next();
			o.complete();
		}
	}


	@Override
	public String getKey() {
		return this.key;
	}

	@Override
	public String getUrl() {
		return this.url;
	}

	@Override
	public void addOutputStreamEntity(String key, AsyncContext entity) throws IOException {
		if (headers == null) {
			outEntitys.add(entity);
		} else {
			entity.getResponse().getOutputStream().write(headers);
			entity.getResponse().getOutputStream().flush();
			outEntitys.add(entity);
		}
	}

	@Override
	public void exit() {
		this.runing = false;
		try {
			this.join();
		} catch (Exception e) {
			log.error(e.getMessage(), e);
		}
	}

}

Controller

@Api(tags = "flv")
@RestController
public class FLVController {

    @Autowired
    private IFLVService service;

    @GetMapping("/openFlv")
    @CrossOrigin
    public void open(HttpServletResponse response,
                      HttpServletRequest request) {
//        String test = "rtsp://***";
        String test = "rtmp://***";
        service.open(test, response, request);
    }

    @GetMapping("/closeFlv")
    @CrossOrigin
    public void close() {
//        String test = "rtsp://***";
        String test = "rtmp://***";
        service.close(test);
    }
}