1、子组件:<ASRDetection/>
<template>
<div>
<div class="message">{{ ASRMessage }}</div>
<div class="wave wave0" :class="isASRAnimation ? 'animation' : ''"></div>
<div class="wave wave1" :class="isASRAnimation ? 'animation' : ''"></div>
<div class="microphone"></div>
</div>
</template>
<script>
import { defineComponent, onMounted, ref, onUnmounted } from 'vue'
import VAD from '@/assets/js/vad.js'
export default defineComponent({
name: 'ASRDetection',
props: {
ASRMessage: {
type: String,
default: '等待说话'
},
videoId: {
type: String,
default: ''
}
},
setup (props, { emit }) {
let localStreamTrack = null
// 控制vad检测动画
let isASRAnimation = ref(false)
let isShow = ref(false)
// let voiceInfo = {
// inputData: new Array(), // 截取的录用数据
// size: 0, // 录音文件大小
// };
// const voiceRecorder = ref();
// 获取截取音频
// const getRecode = () => {
// if (0 !== voiceInfo.size) {
// // 输出pcm二进制数据
// const res = getEncodePCM(voiceInfo.inputData, voiceInfo.size)
// voiceInfo.inputData = [];
// voiceInfo.size = 0;
// return res;
// }
// };
let timer1 = null
let timer2 = null
const initVAD = () => {
window.AudioContext = window.AudioContext || window.webkitAudioContext
var audioContext
// Ask for audio device
navigator.getUserMedia = navigator.mediaDevices.getUserMedia || navigator.mediaDevices.mozGetUserMedia || navigator.mediaDevices.webkitGetUserMedia
navigator.getUserMedia({ audio: true }).then(stream => {
audioContext = new AudioContext()
localStreamTrack = stream
var source = audioContext.createMediaStreamSource(stream)
// console.log()
// voiceRecorder.value = audioContext.createScriptProcessor(4096, 1, 1);
// source.connect(voiceRecorder.value);
// voiceRecorder.value.connect(audioContext.destination);
// voiceRecorder.value.onaudioprocess = function (
// audioProcessingEvent
// ) {
// // getChannelData返回Float32Array类型的pcm数据
// let inputBuffer = audioProcessingEvent.inputBuffer;
// let data = inputBuffer.getChannelData(0);
// voiceInfo.inputData.push(new Float32Array(data));
// voiceInfo.size += data.length;
// };
var options = {
source: source,
voice_start () {
// isASRAnimation.value = true
// isShow.value=true
// emit('voice_start')
console.log('====voice_start',source)
},
voice_stop () {
console.log('=====voice_stopvoice_stopvoice_stopvoice_stopvoice_stopvoice_stopvoice_stopvoice_stop')
setTimeout(() => {
isASRAnimation.value = false
isShow.value = false
emit('voice_stop')
}, 1000);
},
}
console.log(options)
new VAD(options)
})
}
let isIPhone = false
onMounted(() => {
initVAD()
// let u = navigator.userAgent
// if (u.indexOf('iPhone') > -1) {
// isIPhone = true
// }
// console.log(isIPhone, 'isIPhone')
// timer1 = setTimeout(() => {
// isASRAnimation.value = true
// isShow.value = true
// emit('voice_start')
// clearTimeout(timer1)
// if (isIPhone) {
// timer2 = setTimeout(() => {
// isASRAnimation.value = false
// isShow.value = false
// emit('voice_stop')
// clearTimeout(timer2)
// }, 5000);
// }
// }, 1200);
// if (!isIPhone) {
// initVAD()
// }
})
onUnmounted(() => {
if (!isIPhone) {
localStreamTrack.getTracks()[0].stop()
}
if (timer1) {
clearTimeout(timer1)
}
if (timer2) {
clearTimeout(timer2)
}
})
return { isASRAnimation, isShow }
}
})
</script>
<style lang="scss" scoped>
.message {
width: 100%;
position: fixed;
top: 32%;
left: 0;
text-align: center;
z-index: 3;
color: #fff;
}
.wave0 {
width: 220px;
height: 220px;
background: rgba(86, 151, 255, 0.16);
position: fixed;
bottom: 210px;
left: 50%;
margin-left: -110px;
z-index: 1;
border-radius: 50%;
}
.wave1 {
width: 200px;
height: 200px;
background: rgba(86, 151, 255, 0.48);
position: fixed;
bottom: 220px;
left: 50%;
margin-left: -100px;
z-index: 2;
border-radius: 50%;
}
.wave0.animation {
animation: twinkling 1s infinite ease-in-out;
}
.wave1.animation {
animation: twinkling 1s infinite ease-in-out;
}
.microphone {
width: 140px;
height: 140px;
position: fixed;
bottom: 250px;
left: 50%;
margin-left: -70px;
z-index: 3;
background-image: url('../../../assets/microphone.png');
background-size: cover;
}
// @keyframes scale0 {
// 0% {
// transform: scale(1);
// }
// 100% {
// transform: scale(1.2);
// }
// }
// @keyframes scale1 {
// 0% {
// transform: scale(1);
// }
// 100% {
// transform: scale(1.1);
// }
// }
@keyframes twinkling {
0% {
opacity: 0.5;
filter: alpha(opacity=50);
-webkit-transform: scale(1);
}
50% {
opacity: 1;
filter: alpha(opacity=100);
-webkit-transform: scale(1.12);
}
100% {
opacity: 0.5;
filter: alpha(opacity=50);
-webkit-transform: scale(1);
}
}
</style>
2、父组件
<ASRDetection
@voice_start="voice_start"
@voice_stop="voice_stop"
v-if="isASRDetectState"
/>
const voice_start = () => {
console.log("====voice_start", voice_start);
// 监听vad开始事件
};
const voice_stop = () => {
console.log("====voice_stop", voice_stop);
// 监听vad停止事件
isASRDetectState.value = false;
Toast.loading({
message: "检测中...",
forbidClick: true,
duration: 0,
});
};
3、vad.js
const VAD = function(options) {
// Default options
this.options = {
fftSize: 512,
bufferLen: 1024,
voice_stop: function() {},
voice_start: function() {},
smoothingTimeConstant: 0.99,
energy_offset: 1e-8, // The initial offset.
energy_threshold_ratio_pos: 2, // Signal must be twice the offset
energy_threshold_ratio_neg: 0.5, // Signal must be half the offset
energy_integration: 1, // Size of integration change compared to the signal per second.
filter: [
{f: 200, v:0}, // 0 -> 200 is 0
{f: 2000, v:1} // 200 -> 2k is 1
],
source: null,
context: null
};
// User options
for(var option in options) {
// options.hasOwnProperty(option)
// Object.prototype.hasOwnProperty.call(options, option)
if(Object.prototype.hasOwnProperty.call(options, option)) {
this.options[option] = options[option];
}
}
// Require source
if(!this.options.source)
throw new Error("The options must specify a MediaStreamAudioSourceNode.");
// Set this.options.context
this.options.context = this.options.source.context;
// Calculate time relationships
this.hertzPerBin = this.options.context.sampleRate / this.options.fftSize;
this.iterationFrequency = this.options.context.sampleRate / this.options.bufferLen;
this.iterationPeriod = 1 / this.iterationFrequency;
var DEBUG = true;
if(DEBUG) console.log(
'Vad' +
' | sampleRate: ' + this.options.context.sampleRate +
' | hertzPerBin: ' + this.hertzPerBin +
' | iterationFrequency: ' + this.iterationFrequency +
' | iterationPeriod: ' + this.iterationPeriod
);
this.setFilter = function(shape) {
this.filter = [];
for(var i = 0, iLen = this.options.fftSize / 2; i < iLen; i++) {
this.filter[i] = 0;
for(var j = 0, jLen = shape.length; j < jLen; j++) {
if(i * this.hertzPerBin < shape[j].f) {
this.filter[i] = shape[j].v;
break; // Exit j loop
}
}
}
}
// console.log(this.filter,'this.filter')
this.setFilter(this.options.filter);
this.ready = {};
this.vadState = false; // True when Voice Activity Detected
// Energy detector props
this.energy_offset = this.options.energy_offset;
this.energy_threshold_pos = this.energy_offset * this.options.energy_threshold_ratio_pos;
this.energy_threshold_neg = this.energy_offset * this.options.energy_threshold_ratio_neg;
this.voiceTrend = 0;
this.voiceTrendMax = 10;
this.voiceTrendMin = -10;
this.voiceTrendStart = 5;
this.voiceTrendEnd = -5;
// Create analyser
this.analyser = this.options.context.createAnalyser();
this.analyser.smoothingTimeConstant = this.options.smoothingTimeConstant; // 0.99;
this.analyser.fftSize = this.options.fftSize;
this.floatFrequencyData = new Float32Array(this.analyser.frequencyBinCount);
// Setup local storage of the Linear FFT data
this.floatFrequencyDataLinear = new Float32Array(this.floatFrequencyData.length);
// Connect this.analyser
this.options.source.connect(this.analyser);
// console.log(this.options.bufferLen,'this.options.bufferLen')
// Create ScriptProcessorNode
this.scriptProcessorNode = this.options.context.createScriptProcessor(this.options.bufferLen, 1, 1);
// console.log(this.scriptProcessorNode,'this.scriptProcessorNode1')
// Connect scriptProcessorNode (Theretically, not required)
this.scriptProcessorNode.connect(this.options.context.destination);
// console.log(this.scriptProcessorNode,'this.scriptProcessorNode2')
// Create callback to update/analyze floatFrequencyData
var self = this;
this.scriptProcessorNode.onaudioprocess = function() {
self.analyser.getFloatFrequencyData(self.floatFrequencyData);
self.update();
self.monitor();
};
// Connect scriptProcessorNode
this.options.source.connect(this.scriptProcessorNode);
// log stuff
this.logging = false;
this.log_i = 0;
this.log_limit = 100;
this.triggerLog = function(limit) {
this.logging = true;
this.log_i = 0;
this.log_limit = typeof limit === 'number' ? limit : this.log_limit;
}
this.log = function(msg) {
if(this.logging && this.log_i < this.log_limit) {
this.log_i++;
console.log(msg);
} else {
this.logging = false;
}
}
this.update = function() {
// Update the local version of the Linear FFT
var fft = this.floatFrequencyData;
for(var i = 0, iLen = fft.length; i < iLen; i++) {
this.floatFrequencyDataLinear[i] = Math.pow(10, fft[i] / 10);
}
this.ready = {};
}
this.getEnergy = function() {
if(this.ready.energy) {
return this.energy;
}
var energy = 0;
var fft = this.floatFrequencyDataLinear;
// console.log(fft,'fftfftfftfftfftfftfftfftfftfftfftfftfft')
// console.log(this.filter,' this.filterthis.filterthis.filterthis.filterthis.filterthis.filterthis.filterthis.filterthis.filter')
for(var i = 0, iLen = fft.length; i < iLen; i++) {
energy += this.filter[i] * fft[i] * fft[i];
}
this.energy = energy;
this.ready.energy = true;
// console.log(this.energy,' this.energythis.energythis.energythis.energythis.energythis.energythis.energy')
return energy;
}
this.monitor = function() {
var energy = this.getEnergy();
var signal = energy - this.energy_offset;
// console.log('a1',signal,this.energy_threshold_pos,-this.energy_threshold_neg)
// console.log('b1',signal > this.energy_threshold_pos,signal < -this.energy_threshold_neg)
if(signal > this.energy_threshold_pos) {
this.voiceTrend = (this.voiceTrend + 1 > this.voiceTrendMax) ? this.voiceTrendMax : this.voiceTrend + 1;
} else if(signal < -this.energy_threshold_neg) {
this.voiceTrend = (this.voiceTrend - 1 < this.voiceTrendMin) ? this.voiceTrendMin : this.voiceTrend - 1;
} else {
// voiceTrend gets smaller
if(this.voiceTrend > 0) {
this.voiceTrend--;
} else if(this.voiceTrend < 0) {
this.voiceTrend++;
}
}
// console.log('a2',energy,signal,this.energy_threshold_pos,this.voiceTrend,this.voiceTrendStart,this.voiceTrendEnd)
// console.log('b2',this.voiceTrend > this.voiceTrendStart,this.voiceTrend < this.voiceTrendEnd)
var start = false, end = false;
if(this.voiceTrend > this.voiceTrendStart) {
// Start of speech detected
start = true;
} else if(this.voiceTrend < this.voiceTrendEnd) {
// End of speech detected
end = true;
}
// Integration brings in the real-time aspect through the relationship with the frequency this functions is called.
var integration = signal * this.iterationPeriod * this.options.energy_integration;
// Idea?: The integration is affected by the voiceTrend magnitude? - Not sure. Not doing atm.
// The !end limits the offset delta boost till after the end is detected.
if(integration > 0 || !end) {
this.energy_offset += integration;
} else {
this.energy_offset += integration * 10;
}
this.energy_offset = this.energy_offset < 0 ? 0 : this.energy_offset;
this.energy_threshold_pos = this.energy_offset * this.options.energy_threshold_ratio_pos;
this.energy_threshold_neg = this.energy_offset * this.options.energy_threshold_ratio_neg;
// console.log('dd1',start,this.vadState,end)
// Broadcast the messages
// let startBtn=false
// let endBtn=false
// setTimeout(() => {
// startBtn=true
// }, 3000);
// setTimeout(() => {
// endBtn=true
// }, 7000);
if(start && !this.vadState) {
this.vadState = true;
this.options.voice_start();
}
if(end && this.vadState) {
this.vadState = false;
this.options.voice_stop();
}
// if(startBtn) {
// console.log('startstartstartstartstartstartstartstartstartstartstartstartstartstartstartstartstartstartstartstartstartstartstartstartstartstartstartstartstartstart')
// this.options.voice_start();
// }
// if(endBtn) {
// console.log('endendendendendendendendendendendendendendendendendendendendendendendendendend')
// this.options.voice_stop();
// }
// console.log('dd2',start,this.vadState,end)
this.log(
'e: ' + energy +
' | e_of: ' + this.energy_offset +
' | e+_th: ' + this.energy_threshold_pos +
' | e-_th: ' + this.energy_threshold_neg +
' | signal: ' + signal +
' | int: ' + integration +
' | voiceTrend: ' + this.voiceTrend +
' | start: ' + start +
' | end: ' + end
);
return signal;
}
}
export default VAD