H5实现实时音频MP3上传至服务端

背景

最近由于项目需求，需要对接广播，实现在业务平台上集成话筒的实时喊话功能。没问题，很简单的SDK集成。然而这个项目我们连个苦逼的乙方都算不上，只能是作为第三方技术支撑，设备方没有SDK提供，
最后只给了两个API：
1、查询设备信息，连接设备信息的接口；
2、接收实时音频文件的传输接口；
然后需要我们通过websocket将实时的音频文件上传至设备厂家的服务器，同时发送设备连接命令，然后实现实时喊话功能。

**what???**没搞过，咋整，作为一个野生的程序员，查资料是必须会的技能。

网上有大神干过了Recorder用于html5录音
我实现的功能也是借鉴于此，不喜勿喷，如有不足之处，还望不吝指出。

实现方法

音频采集

还是和大神的方法一样，前端使用h5作音频采集，然后实时推送给后台自己的服务器
采集方法：

rec.open(function(){//打开麦克风授权获得相关资源
		clearTimeout(t);
		rec.start();//开始录音
		wave=Recorder.FrequencyHistogramView({elem:".recwave"});
	},function(msg,isUserNotAllow){
		clearTimeout(t);
		console.info((isUserNotAllow?"UserNotAllow，":"")+"无法录音:"+msg, 1);
	});

然后就是websocket连接了

var ws = null; //实现WebSocket
function useWebSocket() {
        ws = new WebSocket("ws://113.57.128.2:980/jetosend/websocket/978d9079-1639-425f-970b-39e06ef24233");
        <!--ws = new WebSocket("ws://192.168.1.103:8080/send/voice");-->
        ws.binaryType = 'arraybuffer'; //传输的是 ArrayBuffer 类型的数据
        ws.onopen = function () {
            console.log('握手成功');
            if (ws.readyState == 1) { //ws进入连接状态，则每隔500毫秒发送一包数据
            	console.log('连接状态成功');
                rec.start();
            }
        };

        ws.onmessage = function (msg) {
            console.info('--------'+msg.data)
        }

        ws.onerror = function (err) {
            console.info('------1111--'+err)
        }

        ws.onclose=function(e){
		  console.info('------2------'+e);
		};

    }

由于是广播，需要实时喊话，当然实时传输就必不可少了

//=====实时处理核心函数==========
var RealTimeSendTry=function(chunkBytes,isClose){
	if(chunkBytes){//推入缓冲再说
		realTimeSendTryBytesChunks.push(chunkBytes);
	};

	var t1=Date.now();
	if(!isClose && t1-realTimeSendTryTime<SendInterval){
		return;//控制缓冲达到指定间隔才进行传输
	};
	realTimeSendTryTime=t1;
	var number=++realTimeSendTryNumber;


	//mp3缓冲的chunk拼接成一个更长点的mp3
	var len=0;
	for(var i=0;i<realTimeSendTryBytesChunks.length;i++){
		len+=realTimeSendTryBytesChunks[i].length;
	};
	var chunkData=new Uint8Array(len);
	for(var i=0,idx=0;i<realTimeSendTryBytesChunks.length;i++){
		var chunk=realTimeSendTryBytesChunks[i];
		chunkData.set(chunk,idx);
		idx+=chunk.length;
	};
	realTimeSendTryBytesChunks=[];

	//推入传输
	var blob=null,meta={};
	if(chunkData.length>0){//mp3不是空的
		blob=new Blob([chunkData],{type:"audio/mp3"});
		meta=Recorder.mp3ReadMeta([chunkData.buffer],chunkData.length)||{};//读取出这个mp3片段信息
	};
	TransferUpload(number
		,blob
		,meta.duration||0
		,{set:{
			type:"mp3"
			,sampleRate:meta.sampleRate
			,bitRate:meta.bitRate
		}}
		,isClose
	);

	realTimeSendTryWavTestBuffers=[];
};

服务器端

这个就简单多了，就是一个简单的ServerSocket，并实现onMessage方法即可

	@OnMessage
    public void onMessage(@PathParam("key")String id, InputStream inputStream) {
        System.out.println("onMessage+" + id);
        try {
        	connections.get(id).getBasicRemote().sendText("接收信息中");
            int rc = 0;
            byte[] buff = new byte[100];
            
            while ((rc = inputStream.read(buff, 0, 100)) > 0) {
                byteArrayOutputStream.write(buff, 0, rc);
               
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

上面的byteArrayOutputStream就是对应的音频流了，再将此流推送到设备厂商的websocket上就OK了，这边就不上这块的代码了，简单的在onClose的时候，将此流存到本地

@OnClose
    public void onClose(@PathParam("key") String id) {
        System.out.println(id + "断开");
        BufferedOutputStream bos = null;
        FileOutputStream fos = null;
        File file = null;
        try {
        	System.out.println(byteArrayOutputStream.toByteArray().length);
            file = new File("D:\\testtest.mp3");
//            file = new File("/opt/nineday/testtest.mp3");
            //输出流
            fos = new FileOutputStream(file);
            //缓冲流
            bos = new BufferedOutputStream(fos);
            //将字节数组写出
            bos.write(byteArrayOutputStream.toByteArray());
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (bos != null) {
                try {
                    bos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (fos != null) {
                try {
                    fos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        try {
			connections.get(id).getBasicRemote().sendText("关闭了");
			connections.remove(id);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
    }

源码

前端页面源码如下：

<!DOCTYPE HTML>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0">

<title>测试音频</title>
<script>

</script>
<!-- 【1.1】引入核心文件 -->
<script src="recorder-core.js"></script>

<!-- 【1.2】引入相应格式支持文件；如果需要多个格式支持，把这些格式的编码引擎js文件放到后面统统加载进来即可 -->
<script src="engine/mp3.js"></script>
<script src="engine/mp3-engine.js"></script>

<!-- 【1.3】引入可选的扩展支持项，如果不需要这些扩展功能可以不引入 -->
<script src="extensions/frequency.histogram.view.js"></script>
<script src="extensions/lib.fft.js"></script>
</head>
<body>

<!-- 【2】构建界面 -->
<div class="main">

	<div class="mainBox">
		<!-- 按钮控制区域 -->
		<div class="pd btns">
			<div>
				<button onclick="recOpen()" style="margin-right:10px">打开录音,请求权限</button>
				<button onclick="recClose()" style="margin-right:0">关闭录音,释放资源</button>
			</div>

			<button onclick="recStart()">录制</button>
			<button onclick="recStop()" style="margin-right:80px">停止</button>

		</div>

		<!-- 波形绘制区域 -->
		<div class="pd recpower">
			<div style="height:40px;width:300px;background:#999;position:relative;">
				<div class="recpowerx" style="height:40px;background:#0B1;position:absolute;"></div>
				<div class="recpowert" style="padding-left:50px; line-height:40px; position: relative;"></div>
			</div>
		</div>
		<div class="pd waveBox">
			<div style="border:1px solid #ccc;display:inline-block"><div style="height:100px;width:300px;" class="recwave"></div></div>
		</div>
	</div>
</div>
<!-- 【3】实现录音逻辑 -->
<script>
var testOutputWavLog=false;//顺带打一份wav的log，录音后执行mp3、wav合并的demo代码可对比音质
var testSampleRate=16000;
var testBitRate=16;

var SendInterval=300;//mp3 chunk数据会缓冲，当pcm的累积时长达到这个时长，就会传输发送。这个值在takeoffEncodeChunk实现下，使用0也不会有性能上的影响。

//重置环境
var RealTimeSendTryReset=function(){
	realTimeSendTryTime=0;
};

var realTimeSendTryTime=0;
var realTimeSendTryNumber;
var transferUploadNumberMax;
var realTimeSendTryBytesChunks = [];
var realTimeSendTryClearPrevBufferIdx;
var realTimeSendTryWavTestBuffers;
var realTimeSendTryWavTestSampleRate;

//=====实时处理核心函数==========
var RealTimeSendTry=function(chunkBytes,isClose){
	if(chunkBytes){//推入缓冲再说
		realTimeSendTryBytesChunks.push(chunkBytes);
	};

	var t1=Date.now();
	if(!isClose && t1-realTimeSendTryTime<SendInterval){
		return;//控制缓冲达到指定间隔才进行传输
	};
	realTimeSendTryTime=t1;
	var number=++realTimeSendTryNumber;


	//mp3缓冲的chunk拼接成一个更长点的mp3
	var len=0;
	for(var i=0;i<realTimeSendTryBytesChunks.length;i++){
		len+=realTimeSendTryBytesChunks[i].length;
	};
	var chunkData=new Uint8Array(len);
	for(var i=0,idx=0;i<realTimeSendTryBytesChunks.length;i++){
		var chunk=realTimeSendTryBytesChunks[i];
		chunkData.set(chunk,idx);
		idx+=chunk.length;
	};
	realTimeSendTryBytesChunks=[];

	//推入传输
	var blob=null,meta={};
	if(chunkData.length>0){//mp3不是空的
		blob=new Blob([chunkData],{type:"audio/mp3"});
		meta=Recorder.mp3ReadMeta([chunkData.buffer],chunkData.length)||{};//读取出这个mp3片段信息
	};
	TransferUpload(number
		,blob
		,meta.duration||0
		,{set:{
			type:"mp3"
			,sampleRate:meta.sampleRate
			,bitRate:meta.bitRate
		}}
		,isClose
	);


	if(testOutputWavLog){
		//测试输出一份wav，方便对比数据
		var recMock2=Recorder({
			type:"wav"
			,sampleRate:testSampleRate
			,bitRate:16
		});
		var chunk=Recorder.SampleData(realTimeSendTryWavTestBuffers,realTimeSendTryWavTestSampleRate,realTimeSendTryWavTestSampleRate);
		recMock2.mock(chunk.data,realTimeSendTryWavTestSampleRate);
		recMock2.stop(function(blob,duration){
			var logMsg="No."+(number<100?("000"+number).substr(-3):number);
			Runtime.LogAudio(blob,duration,recMock2,logMsg);
		});
	};
	realTimeSendTryWavTestBuffers=[];
};

//=====实时处理时清理一下内存（延迟清理），本方法先于RealTimeSendTry执行======
var RealTimeOnProcessClear=function(buffers,powerLevel,bufferDuration,bufferSampleRate,newBufferIdx,asyncEnd){
	if(realTimeSendTryTime==0){
		realTimeSendTryTime=Date.now();
		realTimeSendTryNumber=0;
		transferUploadNumberMax=0;
		realTimeSendTryBytesChunks=[];
		realTimeSendTryClearPrevBufferIdx=0;
		realTimeSendTryWavTestBuffers=[];
		realTimeSendTryWavTestSampleRate=0;
	};

	//清理PCM缓冲数据，最后完成录音时不能调用stop，因为数据已经被清掉了
	//这里进行了延迟操作（必须要的操作），只清理上次到现在的buffer
	for(var i=realTimeSendTryClearPrevBufferIdx;i<newBufferIdx;i++){
		buffers[i]=null;
	};
	realTimeSendTryClearPrevBufferIdx=newBufferIdx;

	//备份一下方便后面生成测试wav
	for(var i=newBufferIdx;i<buffers.length;i++){
		realTimeSendTryWavTestBuffers.push(buffers[i]);
	};
	realTimeSendTryWavTestSampleRate=bufferSampleRate;
};

//=====数据传输函数==========
var TransferUpload=function(number,blobOrNull,duration,blobRec,isClose){
	transferUploadNumberMax=Math.max(transferUploadNumberMax,number);
	if(blobOrNull){
		var blob=blobOrNull;

		//*********Read As Base64***************
		var reader=new FileReader();
		reader.onloadend=function(){
			var base64=(/.+;\s*base64\s*,\s*(.+)$/i.exec(reader.result)||[])[1];

			//可以实现
			//WebSocket send(base64) ...
			//WebRTC send(base64) ...
			//XMLHttpRequest send(base64) ...
			//这里啥也不干
		};
		reader.readAsDataURL(blob);

		//*********Blob***************
		//可以实现
		ws.send(blob);
		//WebSocket send(blob) ...
		//WebRTC send(blob) ...
		//XMLHttpRequest send(blob) ...

		//这里仅 console send 意思意思
		var numberFail=number<transferUploadNumberMax?'<span style="color:red">顺序错乱的数据，如果要求不高可以直接丢弃，或者调大SendInterval试试</span>':"";
		var logMsg="No."+(number<100?("000"+number).substr(-3):number)+numberFail;

		//Runtime.LogAudio(blob,duration,blobRec,logMsg);

		if(true && number%100==0){//emmm....
			Runtime.LogClear();
		};
	};

	if(isClose){
		ws.close();
		console.info("No."+(number<100?("000"+number).substr(-3):number)+":已停止传输");
	};
};



//调用录音
var rec;
var wave=null;
function recStart(){
	if(rec){
		rec.close();
	};
	rec=Recorder({
		type:"mp3"
		,sampleRate:testSampleRate
		,bitRate:testBitRate
		,onProcess:function(buffers,powerLevel,bufferDuration,bufferSampleRate,newBufferIdx,asyncEnd){
			wave.input(buffers[buffers.length-1],powerLevel,bufferSampleRate);
		}
		,takeoffEncodeChunk:function(chunkBytes){
			//接管实时转码，推入实时处理
			RealTimeSendTry(chunkBytes,false);
		}
	});

	var t=setTimeout(function(){
		console.info("无法录音：权限请求被忽略（超时假装手动点击了确认对话框）",1);
	},8000);

	rec.open(function(){//打开麦克风授权获得相关资源
		clearTimeout(t);
		rec.start();//开始录音

		useWebSocket();
		wave=Recorder.FrequencyHistogramView({elem:".recwave"});
		RealTimeSendTryReset();//重置
	},function(msg,isUserNotAllow){
		clearTimeout(t);
		console.info((isUserNotAllow?"UserNotAllow，":"")+"无法录音:"+msg, 1);
	});
};
function recStop(){
	rec.close();//直接close掉即可，这个例子不需要获得最终的音频文件

	RealTimeSendTry(null,true);//最后一次发送
};

var ws = null; //实现WebSocket
function useWebSocket() {
        ws = new WebSocket("ws://192.168.1.103:8080/send/voice");
        ws.binaryType = 'arraybuffer'; //传输的是 ArrayBuffer 类型的数据
        ws.onopen = function () {
            console.log('握手成功');
            if (ws.readyState == 1) { //ws进入连接状态，则每隔500毫秒发送一包数据
            	console.log('连接状态成功');
                rec.start();
            }
        };

        ws.onmessage = function (msg) {
            console.info('--------'+msg.data)
        }

        ws.onerror = function (err) {
            console.info('------1111--'+err)
        }

        ws.onclose=function(e){
		  console.info('------2------'+e);
		};

    }

</script>

<script>
if(/mobile/i.test(navigator.userAgent)){
	//移动端加载控制台组件
	var elem=document.createElement("script");
	elem.setAttribute("type","text/javascript");
	elem.setAttribute("src","https://cdn.bootcss.com/eruda/1.5.4/eruda.min.js");
	document.body.appendChild(elem);
	elem.onload=function(){
		eruda.init();
	};
};
</script>


<style>
body{
	word-wrap: break-word;
	background:#f5f5f5 center top no-repeat;
	background-size: auto 680px;
}
pre{
	white-space:pre-wrap;
}
a{
	text-decoration: none;
	color:#06c;
}
a:hover{
	color:#f00;
}

.main{
	max-width:700px;
	margin:0 auto;
	padding-bottom:80px
}

.mainBox{
	margin-top:12px;
	padding: 12px;
	border-radius: 6px;
	background: #fff;
	--border: 1px solid #f60;
	box-shadow: 2px 2px 3px #aaa;
}


.btns button{
	display: inline-block;
	cursor: pointer;
	border: none;
	border-radius: 3px;
	background: #f60;
	color:#fff;
	padding: 0 15px;
	margin:3px 20px 3px 0;
	line-height: 36px;
	height: 36px;
	overflow: hidden;
	vertical-align: middle;
}
.btns button:active{
	background: #f00;
}

.pd{
	padding:0 0 6px 0;
}
.lb{
	display:inline-block;
	vertical-align: middle;
	background:#00940e;
	color:#fff;
	font-size:14px;
	padding:2px 8px;
	border-radius: 99px;
}
</style>

</body>
</html>

服务端源码

@ServerEndpoint("/send/{key}")
@Component
public class ServerSocket {

    private static final Map<String, Session> connections = new Hashtable<>();
    private static ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();

    /***
     * @Description:打开连接
     * @Param: [id, 保存对方平台的资源编码
     * session]
     * @Return: void
     * @Author: ZCH
     * @Date: 2021-01-10 09:02
     */
    @OnOpen
    public void onOpen(@PathParam("key") String id, Session session) {
        try {
			System.out.println(id + "连上了");
			connections.put(id, session);
			connections.get(id).getBasicRemote().sendText("连接上了");
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
    }

    /**
     * 接收消息
     */
    @OnMessage
    public void onMessage(@PathParam("key")String id, InputStream inputStream) {
        System.out.println("onMessage+" + id);
        try {
        	connections.get(id).getBasicRemote().sendText("接收信息中");
            int rc = 0;
            byte[] buff = new byte[100];
            while ((rc = inputStream.read(buff, 0, 100)) > 0) {
                byteArrayOutputStream.write(buff, 0, rc);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 异常处理
     *
     * @param throwable
     */
    @OnError
    public void onError(Throwable throwable) {
        throwable.printStackTrace();
        //TODO 日志打印异常
    }

    /**
     * 关闭连接
     */
    @OnClose
    public void onClose(@PathParam("key") String id) {
        System.out.println(id + "断开");
        BufferedOutputStream bos = null;
        FileOutputStream fos = null;
        File file = null;
        try {
        	System.out.println(byteArrayOutputStream.toByteArray().length);
            file = new File("D:\\testtest.mp3");
//            file = new File("/opt/nineday/testtest.mp3");
            //输出流
            fos = new FileOutputStream(file);
            //缓冲流
            bos = new BufferedOutputStream(fos);
            //将字节数组写出
            bos.write(byteArrayOutputStream.toByteArray());
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (bos != null) {
                try {
                    bos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (fos != null) {
                try {
                    fos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        try {
			connections.get(id).getBasicRemote().sendText("关闭了");
			connections.remove(id);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
    }
}