javacv音视频截图,音频抽取汇总

概述:

java 使用 javacv 视频截帧, 跳到指定帧截图, 从视频中抽取音频, 音频分割为多段

1 依赖:

<dependency>
 <groupId>org.bytedeco</groupId>
 <artifactId>javacv</artifactId>
 <version>1.5.7</version>
</dependency>

<dependency>
 <groupId>org.bytedeco</groupId>
 <artifactId>javacv-platform</artifactId>
 <version>1.5.7</version>
</dependency>

注意: javacv中mat与tf模型中mat格式不兼容。如果需要调用TensorFlow等传Mat，需要将javacv中的opencv排掉，引nexus上opencv包并加载opencv lib包。

2 视频截帧：

1) 视频连续截帧/间隔截帧

此种方式为从头开始循环视频每一帧, 适用于连续或如每10帧截一帧图等比较密集的连续截帧.

//@Param : path:视频path; dir:截帧保存文件夹,也可以直接使用mat不保存图片; frameInterval:每多少帧截一帧
public void grabFrameInterval(String path, String dir, int frameInterval) throws Exception {
 logger.info("视频截帧 source sourceFile: {}", path);
 try (FFmpegFrameGrabber frameGrabber = new FFmpegFrameGrabber(path)) {
 frameGrabber.start();
 int lengthInFrames = frameGrabber.getLengthInFrames();
 logger.info("视频截帧 开始, 视频共 {} 帧, 预计截帧: {}张", lengthInFrames, lengthInFrames/frameInterval);

 int frameCount = -1; //第一帧不截, 从第二帧开始截
 Frame frame;
 while ((frame = frameGrabber.grabImage()) != null) {
 frameCount++;
 String imagePath = dir + frameCount+".jpg";
 if (frameCount % frameInterval != 0) continue;
 BufferedImage bufferedImage = converter.getBufferedImage(frame);
 try (FileOutputStream fileOutputStream = new FileOutputStream(imagePath)) {
 ImageIO.write(bufferedImage, "jpg", fileOutputStream);
 }
 }
 logger.info("视频截帧 完成, 视频共 {} 帧, 截帧: {}张", lengthInFrames, frameCount+1);
 }
}

注意: 在javacv中有 org.opencv.videoio.VideoCapture 也可视频截帧, 但在centos环境open video file为false, 无法运行, 即使安装opencv关联ffmpeg也不行, 暂时无法使用, 大佬们如果可以解决敬请留言!

//centos 不可用
VideoCapture videoCapture = new VideoCapture();
boolean open = videoCapture.open(videoFile);

2) 视频跳帧截图

此种方式为跳到视频指定帧截图, 适用于要截的帧比较稀疏, 用连续截帧会很慢, 如算法识别到关键帧后, 直接跳到该帧截图

/**
 * ※Desc : 跳帧截图;
 * ※Date : 2022/4/2
 * ※Param : path:视频path; dir:截帧保存文件夹; frameIndexes:需要截帧的FrameNumber
 */
public void grabFrameJump(String path, String dir, List<Integer> frameIndexes) throws Exception {
 logger.info("视频截帧 sourceFile: {}", path);

 try (FFmpegFrameGrabber frameGrabber = new FFmpegFrameGrabber(path)) {
 frameGrabber.stop();
 frameGrabber.start();

 long lengthInTime = frameGrabber.getLengthInTime();
 logger.info("视频截帧 lengthInTime: {}" , lengthInTime);

 for (Integer frameIndex : frameIndexes) {
 frameGrabber.setVideoFrameNumber(frameIndex);
 Frame frame = frameGrabber.grabImage();
 String imageName = frameIndex+".jpg";
 String imagePath = dir + imageName;

 BufferedImage bufferedImage = converter.getBufferedImage(frame);
 try (FileOutputStream fileOutputStream = new FileOutputStream(imagePath)) {
 ImageIO.write(bufferedImage, "jpg", fileOutputStream);
 }
 }
 frameGrabber.stop();
 }
 logger.info("视频截帧 source sourceFile: {}", path);
}

其中frameGrabber.setVideoFrameNumber(frameIndex);可以直接跳到指定帧; 有 frameGrabber.setTimestamp 方法跳到指定时间戳, 但现版本会卡死, 大佬们如果可以解决敬请留言!

3 音频抽取：

1) 视频抽取音频

/**
 * ※Desc : 传入视频文件分离出语音文件
 * ※Author : zhangsong@eastmoney.com
 * ※Date : 2022/3/8
 * ※Param : sourceFile 视频文件(mp4); destDir 转音频后存放文件目录, 文件名为 原文件名+ .mp3
 * ※Return :
 */
public static void extractVoice(String sourceFileName, String destFileName) {
 logger.info("extractVoice source sourceFile: {}, destFileName: {}", sourceFileName, destFileName);
 //抓取资源
 FFmpegFrameGrabber frameGrabber = new FFmpegFrameGrabber(sourceFileName);
 Frame frame = null;
 FFmpegFrameRecorder recorder = null;
 try {
 frameGrabber.start();
 //转录为单轨, 16K采样率, wav格式
 recorder = new FFmpegFrameRecorder(destFileName, 1);//frameGrabber.getAudioChannels()
 recorder.setFormat(Constants.AUDIO_FORMAT);
 recorder.setSampleRate(Constants.AUDIO_SampleRate);//frameGrabber.getSampleRate()
 //recorder.setAudioBitrate(128000);// 音频比特率
 recorder.setTimestamp(frameGrabber.getTimestamp());
 logger.info("source video AudioChannels: {}, SampleRate: {}, AudioBitrate: {}", frameGrabber.getAudioChannels(), frameGrabber.getSampleRate(), recorder.getAudioBitrate());

 recorder.start();
 int index = 0;
 while (true) {
 frame = frameGrabber.grabSamples();
 if (frame == null) break;
 if (frame.samples != null) {
 recorder.recordSamples(frame.sampleRate, frame.audioChannels, frame.samples);
 recorder.setTimestamp(frameGrabber.getTimestamp());
 }
 index++;
 }
 recorder.stop();
 recorder.release();
 frameGrabber.stop();
 logger.info("extractVoice 完成 index=" + index);
 } catch (Exception e) {
 e.printStackTrace();
 }
}

注: 只要抓取音频用 frameGrabber.grabSamples(); 用 frameGrabber.grab() 含有图片等信息会慢很多.

2) 音频截段

如下方法为将一段音频按长度截为多段, 需要截取指定时间段音频也可参考此方法

/**
 * ※Desc : 对给定的音频文件截取, 输出到destDir
 * ※Author : zhangsong@eastmoney.com
 * ※Date : 2022/3/8
 * ※Param : sourceFileName: 源音频文件; audioSplitLength: 多长时间截一段
 */
public static List<AudioPartInfo> splitAudio(String sourceFileName, String destDir, Long audioSplitLength) {
 FFmpegFrameGrabber frameGrabber = null;
 List<AudioPartInfo> list = new ArrayList<>();
 try {
 FileUtils.forceMkdir(new File(destDir));
 logger.info("splitAudio source sourceFile: {}, destDir: {}", sourceFileName, destDir);
 // 创建文件抓取器, 使用javacv的FFmpeg的相关工具包
 frameGrabber = new FFmpegFrameGrabber(sourceFileName);
 //frameGrabber.setOption("RW_TIMEOUT", "20000000");
 Frame audioSamples = null; // 默认音频是pcm采样数据
 // 输出文件
 if (start(frameGrabber)) {
 // 时间长度,微秒
 long lengthInTimeMs = frameGrabber.getLengthInTime();
 long part = lengthInTimeMs / audioSplitLength + 1;
 logger.info("lengthInTimeMs: {}, splitAudio part: {}", lengthInTimeMs, part);

 int partIndex = -1;
 FFmpegFrameRecorder recorder = null;
 AudioPartInfo audioPartInfo = null;
 while ((audioSamples = frameGrabber.grabSamples()) != null) {
 long timestamp = frameGrabber.getTimestamp();
 if (timestamp >= (partIndex+1) * Constants.AUDIO_SPLIT_LENGTH) {
 if (recorder != null) stop(recorder);
 partIndex++;
 //logger.info("开始截取第{}段音频, start timestamp: {}", partIndex, timestamp);
 String audioPath = destDir + partIndex + Constants.AUDIO_SUFFIX;
 recorder = new FFmpegFrameRecorder(audioPath , frameGrabber.getAudioChannels());
 // 编码格式
 recorder.setFormat(Constants.AUDIO_FORMAT);
 recorder.setAudioCodec(frameGrabber.getAudioCodec()); // 65543
 recorder.setAudioBitrate(frameGrabber.getAudioBitrate());
 recorder.setSampleRate(frameGrabber.getSampleRate());
 recorder.setAudioQuality(0);// 最高质量
 recorder.start();
 if (audioPartInfo!=null) audioPartInfo.setEnd(timestamp / 1000); //上一段语音的结束时间
 audioPartInfo = new AudioPartInfo(partIndex, audioPath);
 audioPartInfo.setStart(timestamp / 1000); //μs转为ms
 list.add(audioPartInfo);
 }
 //timestamp 大于 截取起始时间 开始录制
 if (recorder != null) {
 recorder.setTimestamp(timestamp);
 recorder.record(audioSamples);
 audioPartInfo.setEnd(timestamp / 1000); //μs转为ms //这里的timestamp没有包含此次grab的结尾!
 }
 }
 if (recorder != null) stop(recorder);
 }
 } catch (Exception e) {
 logger.error("", e);
 }
 stop(frameGrabber);
 logger.info("splitAudio audioPartInfoList: {}", JSON.toJSONString(list));
 return list;
}