学习笔记-webrtc

概述

• 音视频直播两种方式
  • 实时互动
    • 会议
    • webRtc
  • 流媒体分发
    • 娱乐直播
    • RTMP,HTTP-FLV,HLS

两种指标

• 延迟指标
• 音视频服务质量指标
  • 分辨率
  • 帧率
  • 码率
  • MOS
    • 服务质量评估

压缩算法

• H264
• H265
• AVI

直播架构

• 音视频采集模块
• 音视频编码模块
• 网络传输模块
• 音视频解码模块
• 音视频渲染模块

WebRtc客户端架构

• 接口层
  • Web接口
    • 浏览器
  • Native接口
    • C++,Android,OC
• Session层
  • 媒体协商
  • 收集Candidate。。。
• 核心引擎层
  • 音频引擎
  • 视频引擎
  • 网络传输
• 设备层
  • 硬件采集和播放

构建一对一信令服务器

• 组成
  • 2个WebRtc终端
  • 1个信令服务器
  • 1个中继服务器(STUN/TURN)
    • 获取各自外网ip和端口
  • 2个NAT

信令设计

• 客户端
  • join
    • 用户加入房间
  • leave
    • 用户离开房间
  • message
    • 发送端到端消息
• 服务器
  • joined
    • 用户已加入
  • leaved
    • 用户已离开
  • other_joined
    • 其他用户加入
  • bye
    • 其他用户离开
  • full
    • 房间已满

代码实现

• socket

  • socket.emit(‘xxx’)
    • 发送消息
  • socket.emit(‘xxx’,xxx1,xx2)
    • 发送带参消息
  • socket.to(room).emit(‘xxx’)
    • 给房间内所有人发消息
  • socket.on(‘xxx’,function(){})
    • 接收消息
  • socket.on(‘xxx’,function(arg){})
    • 接收带参数消息

• HTTP服务器

const http = require('http')
const express = require('express')

const app = express()
const http_server = http.createServer(app)

http_server.listen(8081,'0.0.0.0')

• 完整代码

'use strict'

const log4js = require('log4js');
const fs = require('fs');
const http = require('http');
const https = require('https');
const {Server} = require('socket.io')
const path = require('path')
const express = require('express');
const app = express();
app.use(express.static(path.join(__dirname, 'public')))
const USERCOUNT = 3;

log4js.configure({
    appenders: {
        file: {
            type: 'file',
            filename: 'app.log',
            layout: {
                type: 'pattern',
                pattern: '%r %p - %m',
            }
        }
    },
    categories: {
        default: {
            appenders: ['file'],
            level: 'debug'
        }
    }
});
let logger = log4js.getLogger();

let options = {
    key:fs.readFileSync('privkey.pem'),
    cert:fs.readFileSync('cacert.pem')
}
let https_server = https.createServer(options, app);
let http_server = http.createServer(app);
https_server.listen(8080, '0.0.0.0');
http_server.listen(8081, '0.0.0.0');
let httpsIo = new Server(https_server);
let httpIo = new Server(http_server);

httpsIo.sockets.on('connection',(socket) =>{
    // console.log("connection")
    socket.on('message',(room,data) =>{
        console.log("message:"+data)
        // 中转消息
        socket.to(room).emit('message',room,data);
    });
    socket.on('join',(room) =>{
        socket.join(room);
        let myRoom=httpsIo.sockets.adapter.rooms[room];
        let users=(myRoom)?Object.keys(myRoom.sockets).length:0;
        logger.info('the user number of room is :'+users);
        if(users < USERCOUNT)
        {
            // 给自己发join
            socket.emit('joined',room,socket.id);
            if(users > 1)
            {
                // 给其他人发other_joined
                socket.to(room).emit('other_joined',room,socket.id);
            }
        }else
        {
            socket.leave(room);
            // 给自己发full
            socket.emit('full',room,socket.id);

        }
    });

    socket.on('level', (room)=> {
        socket.leave(room);
        // let myRoom=io.sockets.adapter.rooms[room];
        // let users=(myRoom)?Object.keys(myRoom.sockets).length:0;
        // 给其他人发bye
        socket.to(room).emit('bye',room,socket.id);
        // 给自己发level
        socket.emit('leaved',room,socket.id);


    });

});

httpIo.sockets.on('connection',(socket) =>{
    socket.on('message',(room,message) =>{
        console.log("message:"+message)
        // 转发给房间内所有人
        socket.to(room).emit('message',message);
    });
    socket.on('join',(room) =>{
        socket.join(room);
        let myRoom=httpIo.sockets.adapter.rooms[room];
        let users=(myRoom)?Object.keys(myRoom.sockets).length:0;
        logger.info('the user number of room is :'+users);
        if(users < USERCOUNT)
        {
            // 给自己发join
            socket.emit('joined',room,socket.id);
            if(users > 1)
            {
                // 给其他人发other_joined
                socket.to(room).emit('other_joined',room,socket.id);
            }
        }else
        {
            // 给自己发full
            socket.emit('full',room,socket.id);
            socket.leave(room);
        }
    });

    socket.on('level', function(room) {
        socket.leave(room);
        // let myRoom=io.sockets.adapter.rooms[room];
        // let users=(myRoom)?Object.keys(myRoom.sockets).length:0;
        // 给其他人发bye
        socket.to(room).emit('bye',room,socket.id);
        // 给自己发level
        socket.emit('left',room,socket.id);
    });

});

浏览器webrtc

遍历音视频设备

• 遍历音视频设备
  • navigater.mediaDevices.enumerateDevices()
  • MediaDeviceInfo
    • deviceId
    • kind
    • label
    • groupId

function handleError(error){
    console.log('err:',error)
}

function gotDevices(deviceInfos){
    for(let i=0;i<deviceInfo.length;i++){
        const deviceInfo = deviceInfos[i]
    }
}

// 遍历
navigator.mediaDevices.enumerateDevices().then(gotDevices).catch(handleError)

采集音视频数据

• 采集音视频数据
  • navigator.mediaDevices.getUserMedia(MediaStreamConstrains)
  • MediaStreamConstrains
    • video:true/false/MediaStreamConstrainsSet
    • audio:true/false/MediaStreamConstrainsSet
• MdeiaStream
  • 流
• MediaStreamTrack
  • 轨

function gotMediaStream(stream){
    
}

let deviceId = 'xxx'

let constraints = {
    videa:{
        width:640,
        height:480,
        frameRate:15,                               // 帧率15帧/秒
        facingMode:'environment',                   // 后置摄像头
        deviceId:deviceId?{exact:deviceId}:undefined
    },
    radio:false
}

// 开始采集数据
navigator.mediaDevices.getUserMedia(constraints).then(gotMediaStream).catch((handleError))

本地视频预览

<video autoplay playsinline></video>

const lv = document.querySelector('video')
const contrains = {
    vide:true,
    audio:true
}

function gotLocalStream(mediaStream){
    lv.srcObject = mediaStream
}

function handleLocalStreamError(error){
    console.log('err:',error)
}

navigator.mediaDevices.getUserMedia(contrains).then(gotLoaclStram).catch(handleLocalStreamError)

信令状态机

• init
• joined
• joined_conn
• joined_unbind

RTCPeerConnection

const configuration = {
    iceServers:[{urls:'stun:stun.example.org'}]
}
let pc = new RTCPeerConnection(configuration)

绑定Track

    // ls getUserMedia()获取到的MediaStream
    function bindTracks(){
        ls.getTracks().forEach((track)=>{
            pc.addTrack(track,ls)
        })
    }

媒体协商

• SDP
• 交换本地软硬件编码相关信息
• createOffer
• setLocalDescription
• sendOffer
• setRemoteDescription

ICE

• 服务器相关信息
• IceCandidate
  • candidate
    • address
    • port
    • protocol
  • sdpMid
  • sdpMLineIndex
• 连接步骤
  • 收集candidate
  • 交换candidate
  • 尝试连接

// 获取本地candidate
pc.onicecandidate = (e)=>{
    if(e.candidate){
        
    }
}

SDP与ICE中转

• 客户端发送

function sendMessage(roomid,data){
    socket.emit('message',roomid,data)
}

• 服务端接收转发

socket.on('message',(room,data) =>{
        // 中转消息
        socket.to(room).emit('message',room,data);
});

• 客户端接收

    // 客户端接收
   socket.on('message',(room,data) =>{
        console.log("message:"+data)
        if(data.hasOwnProperty('type')&&data.type==='offer'){
            
        }else if(data.hasOwnProperty('type')&&data.type==='answer'){
            
        }else if(data.hasOwnProperty('type')&&data.type==='candidate'){
            
        }else{
           
        }
    });

远端获取音视频流

function getRemoteStream(e){
    
}
let pc = new RTCPeerConnection(...)
pc.ontrack = getRemoteStream()

完成代码

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>WebRtc</title>
    <link href="css/client2.css" rel="stylesheet">

</head>
<body>
<div>
    <div>
        <button id="connserver">ConnServer</button>
        <button id="leave" disabled>Leave</button>
    </div>
    <div id="preview">
        <div>
            <h2>Local</h2>
            <video id="localVideo" autoplay playsinline muted></video>
            <h2>Offer SDP</h2>
            <textarea id="offer"></textarea>
        </div>
        <div>
            <h2>Remote</h2>
            <video id="remoteVideo" autoplay playsinline></video>
            <h2>Answer SDP</h2>
            <textarea id="answer"></textarea>
        </div>
    </div>
</div>
<!-- <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/2.0.3/socket.io.js"/> -->
<!-- <script src="https://webrtc.github.io/adapter/adapter-lastest.js"/> -->

<script src="./js/socket_io.js"></script>
<script src="./js/adapter.js"></script>
<script src="./js/client2.js"></script>

</body>
</html>

   

button{
    margin: 10px 20px 25px 0;
    vertical-align: top;
    width: 134px;
}

/*table{*/
/*    margin: 200px (50% - 100px) 0 0;*/
/*}*/

textarea{
    color:#444;
    font-size: 0.9em;
    font-weight: 300;
    height: 20.0em;
    padding: 5px;
    width: calc(100% - 10px);
}

div#getUserMedia{
    padding: 0 0 8px 0;
}

div#preview{
    border-bottom: 1px solid #eee;
    margin: 0 0 1em 0;
    padding: 0 0 0.5em 0;
}

div#preview>div{
    display: inline-block;
    vertical-align: top;
    width: calc(50% - 12px);
}

video{
    background: #222;
    margin: 0 0 0 0;
    --width:100%;
    width: var(--width);
    height: 225px;
}

@media screen and (max-width:720px){
    button{
        font-weight: 500;
        height: 56px;
        line-height: 1.3em;
        width: 90px;
    }
    div#getUserMedia{
        padding: 0 0 40px 0;
    }

}

`use strict`

var localVideo = document.querySelector('video#localVideo')
var remoteVideo = document.querySelector('video#remoteVideo')

var btnConn = document.querySelector('button#connserver')
var btnLeave = document.querySelector('button#leave')

var offer = document.querySelector('textarea#offer')
var answer = document.querySelector('text#answer')




let pcConfig = {
    iceServers:[{urls:'stun:stun.l.google.com:19302'}]
}

let localStream = null
let remoteStream = null

let pc = null

let roomid = null
let socket = null

let offerdesc = null;
let state = 'init'

function isPc(){
    let userAgentInfo = navigator.userAgent
    let agents = ['Android','iPhone','iPad','iP']
    let flag = true
    for(let i=0;i<agents.length;i++){
        if(userAgentInfo.indexOf(agents[i])>0){
            flag = false
            break
        }
    }
    return flag
}

function isAndroid(){
    let u = navigator.userAgent
    let app = navigator.appVersion
    let isAndroid = u.indexOf('Android') > -1 || u.indexOf('Linux') > -1
    let isIOS = !!u.match(/\(i[^;]+;( U;)? CPU.+Mac OS X/)
    if(isAndroid){
        return true;
    }
    if(isIOS){
        return false
    }
}

function getQueryVariable(variable){
    let query = window.location.search.substring(1)
    let vars = query.split("&")
    for(let i=0;i<vars.length;i++){
        let pair = vars[i].split("=")
        if(pair[0]===variable){
            return pair[1]
        }
    }
    return false
}

function sendMessage(roomid,data){
    console.log('send message to other end',roomid,data)
    if(!socket){
        console.log("socket is null")
    }
    socket.emit('message',roomid,data)
}

function conn(){
    // 连接信令服务器
    socket = io.connect("https://192.168.1.9:8080")
    socket.on('joined',(roomid,id)=>{
        console.log('receive joined message',roomid,id)
        // 状态机变更
        state = 'joined'
        // 创建PeerConnection并绑定音视频轨
        createPeerConnection()
        bindTracks()
        // 设置button状态
        btnConn.disabled = true
        btnLeave.disabled = false
        console.log('reveive joined message,state=',state)
    })
    socket.on('other_joined',(roomid)=>{
        console.log('receive joined message:',roomid,state)
        if(state==='joined_unbind'){
            createPeerConnection()
            bindTracks()
        }
        // 更新状态机
        state = 'joined_conn'
        // 开始呼叫对方
        call()
        console.log('receive other_join message,state=',state)
    })

    socket.on('full',(roomid,id)=>{
        console.log('receive fuu message',roomid,id)
        socket.disconnect()
        // 挂断
        hangup()
        // 关闭本地媒体
        closeLocalMedia()
        // 状态机变更
        state = 'leaved'
        console.log('receive full message,state=',state)
        alert('this room is full')
    })

    socket.on('leaved',(roomid,id)=>{
        console.log('receive leaved message',roomid,id)
        // 状态机变更
        state = 'leaved'
        socket.disconnect()
        console.log('receive leaved message,state=',state)

        // 改变button状态
        btnConn.disabled = false
        btnLeave.disabled = true
    })

    socket.on('bye',(roomid,id)=>{
        console.log('receive bye message',roomid,id)
        // 状态机变更
        state = 'joined_unbind'
        // 挂断
        hangup()

        offer.value = ''
        answer.value = ''
        console.log('receive bye message,state=',state)
    })

    socket.on('disconnect',(socket)=>{
        console.log('receive disconnect message',roomid)
        if(!(state==='leaved')){
            hangup()
            closeLocalMedia()
        }
        state = 'leaved'
    })

    socket.on('message',(roomid,data) =>{
        console.log("receive message!",roomid,data)
        if(data===null||data===undefined){
            console.log('the message is invalid')
            return
        }
        if(data.hasOwnProperty('type')&&data.type==='offer'){
            // 收到SDP是offer
            offer.value = data.sdp
            // 进行媒体协商
            pc.setRemoteDescription(new RTCSessionDescription(data))
            // 创建answer
            pc.createAnswer().then(getAnswer).catch(handleAnswerError)
        }else if(data.hasOwnProperty('type')&&data.type==='answer'){
            // 收到SDP是answer
            answer.value = data.sdp
            // 进行媒体协商
            pc.setRemoteDescription(new RTCSessionDescription(data))
        }else if(data.hasOwnProperty('type')&&data.type==='candidate'){
            // 收到candidate
            let candidate = new RTCIceCandidate({sdpMLineIndex:data.label,candidate:data.candidate})
            // 将远端Candidate消息添加到PeerConnection
            pc.addIceCandidate(candidate)
        }else{
           console.log('the message is invalid',data)
        }
    });
    roomid = getQueryVariable('room')
    socket.emit('join',roomid)
    return true
}



// 打开音视频成功的回调函数
function getMediaStream(stream){
    if(localStream){
        stream.getAudioTracks().forEach((track)=>{
            localStream.addTrack(track)
            stream.removeTrack(track)
        })
    }else{
        localStream = stream
    }
    localVideo.srcObject = localStream
    conn()
}
// 错误处理函数
function handleError(err){
    console.log('Failed to get Media Stream',err)
}

function start(){
    if(!navigator.mediaDevices||!navigator.mediaDevices.getUserMedia){
        console.log('the getUserMedia is not support')
        return
    }else{
        let constraints = {
            video:true,
            audio:{
                echoCancellation:true,
                noiseSuppression:true,
                autoGainControl:true
            }
        }
        navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).catch(handleError)
    }

}
// 获取远端媒体流
function getRemoteStream(e){
    remoteStream = e.streams[0]
    remoteVideo.srcObject = e.streams[0]
}

// 处理Offer错误
function handleOfferError(err){
    console.log('Failed to create offer',err)
}
// 处理answer错误
function handleAnswerError(err){
    console.log('Failed to create answer',err)
}

// 获取answer sdp
function getAnswer(desc){
    answer.value = desc.sdp
    sendMessage(roomid,desc)
}

// 获取offer sdp
function getOffer(dess){
    pc.setLocalDescription(desc)
    offer.value = desc.sdp
    offerdesc = desc
    sendMessage(roomid,offerdesc)
}

function createPeerConnection(){
    console.log('create PeerConnection')
    if(!pc){
        pc = new RTCPeerConnection(pcConfig)
        // 当收集到Candidate
        pc.onicecandidate = (e)=>{
            if(e.candidate){
                console.log("candidate "+ JSON.stringify(e.candidate.toJSON()))
                sendMessage(roomid,{
                    type:'candidate',
                    label:event.candidate.sdpMLineIndex,
                    id:event.candidate.sdpMid,
                    candidate:event.candidate.candidate
                })
            }else{
                console.log('this is the end candidate')
            }
        }
        pc.ontrack = getRemoteStream
    }else{
        console.log('the pc have be created')
    }
    return
}

// 将音视频track绑定到PeerConnection对象中
function bindTracks(){
    console.log('bind tracks into RTCPeerConnection')
    if(pc===null&&localStream==undefined){
        console.log('ps ic null or undefined')
        return
    }
    // 将本地音视频流添加到RTCPeerConnection
    localStream.getTracks().forEach((track)=>{
        pc.addTrack(track,localStream)
    })
}

// 呼叫
function call(){
    if(state==='joined_conn'){
        let offerOptions= {offerToReceiveVideo:1,offerToReceiveAudio:1}
        pc.createOffer(offerOptions).then(getOffer).catch(handleOfferError)
    }
}
// 挂断
function hangup(){
    if(!pc){
        return
    }
    offerdesc = null
    pc.close()
    pc = null
}

// 关闭本地媒体
function closeLocalMedia(){
    if(!(localStream===null||localStream===undefined)){
        localStream.getTracks().forEach((track)=>{
            track.stop()
        })
    }
    localStream = null
}




// 打开音视频设备，连接信令服务器
function starConn(){
    // 开启本地视频
    start()
    return true
}

// 离开
function leave(){
    socket.emit('leave',roomid)
    hangup()
    closeLocalMedia()
    offer.value=''
    answer.value= ''
    btnConn.disabled = false
    btnLeave.disabled = true
}

btnConn.onclick = starConn
btnLeave.onclick = leave