【UVCcamera/新手向】简单的网络摄像头(webcam)

• 说在前面

Android Studio版本：3.2

操作系统（AS）：Windows10

Android版本：5.0.2

摄像头：USB摄像头（640*480 贼辣鸡）

服务器端：NodeJS

通信方法：Socket.IO

开源项目：saki4510t/UVCCamera

接上篇：【UVCcamera/新手向】AS中使用OpenCV进行简单人脸识别

• 一些说明

大致流程

    1.  Android端获取uvccamera数据（bitmap格式）

    2.  将每一帧的数据压缩为JPEG格式的byte[]

    3.  通过socketio将数据发送至server

    4.  server转发这些数据到页面

    5.  接收到数据后将其处理成jpeg图片并显示

一连串的jpeg就成了视频了（我哭了，不是视频流，也没有进行优化，我还是太菜了）

UVCcamera相关请参考第一篇（https://blog.csdn.net/qq_33446100/article/details/88775629） ，涉及部分后台相关代码，后续会将项目上传至Github.

server端放在本地，监听本地IP以及端口。

• 运行截图

• 代码说明

1.  Android端 Socket.IO配置

在app目录下的build.gradle文件中dependencies添加

implementation ('io.socket:socket.io-client:0.8.3') {
        // excluding org.json which is provided by Android
        exclude group: 'org.json', module: 'json'
    }

在mainActivity.java中导入包

import io.socket.client.IO;
import io.socket.client.Socket;

 定义变量

Socket mSocket;

 初始化URL中IP为nodejs所在主机的内网IP（windows在cmd窗口使用ipconfig命令查看，linux使用ifconfig命令）,8081为监听的端口号。

public static final String CHAT_SERVER_URL = "http://192.168.3.12:8081";

//这部分放在OnCreate方法里
        try {
            mSocket = IO.socket(CHAT_SERVER_URL);

        } catch (URISyntaxException e) {
            Log.v(TAG, "Socket Failed");
            throw new RuntimeException(e);

        }

连接server，这个我放在OnStart方法里，其他地方应该也行；

mSocket.connect();

2.  Android端处理UVCcamera捕捉到的图像，并通过SocketIO转发至server

这部分和上一篇的Opencv处理那里有点相似，主要更改的地方也是那一块

获取每一帧的bitmap，压缩为jpeg格式并转换为byte[]

    private void sendBitmap(Bitmap it){

        //Bitmap bmp = getTransformBitmap(it);//这个函数本来是用来将RGB565转为
        //ARGB8888的，但是发现RGB565也可以，就去掉了
        ByteArrayOutputStream buf;
        buf = new ByteArrayOutputStream();
        it.compress(Bitmap.CompressFormat.JPEG, 20, buf);//压缩为JPEG格式
        //byte[] b = buf.toByteArray();
        mSocket.emit("image", buf.toByteArray());//发送（“event”,"value"）

        //Log.v(TAG, "In this module");//测试用
    }

在IFrameCallback函数中调用sendBitmap，也就是上一篇opencv处理那里

sendBitmap(srcBitmap);

3.  配置server

有关nodejs部分的基础知识啥的咱就不说了，自己去找，推荐一个我学习的（Nodejs学习笔记（七）--- Node.js + Express 构建网站简单示例）

app.js(这部分写的有点复杂了，因为是用的之前项目的代码)

var createError = require('http-errors');
var express = require('express');
var path = require('path');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var logger = require('morgan');
var moment = require('moment');

var indexRouter = require('./routes/index');

var app = express();

// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'ejs');

app.use(logger('dev'));
app.use(express.json());
app.use(express.urlencoded({ extended: false }));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));

app.use('/', indexRouter);

// catch 404 and forward to error handler
app.use(function (req, res, next) {
  next(createError(404));
});

// error handler
app.use(function (err, req, res, next) {
  // set locals, only providing error in development
  res.locals.message = err.message;
  res.locals.error = req.app.get('env') === 'development' ? err : {};

  // render the error page
  res.status(err.status || 500);
  //res.render('error');
});

var server = app.listen(8081, function () {
  console.log("Server Start!");
});

var count = 0;
var interval = 50000;
var totalCount = 0;
var allMem = new Array();

var io = require('socket.io')(server);

io.on('connection', function (socket) {
  console.log("New Socket!");

  console.log("当前有用户连接");
  count++;
  totalCount++;
  console.log("count:" + count);

  let name = '';

  //处理image事件
  socket.on("image", function (message) {
    //console.log(message);
    interval++;
    // 转发给所有用户，message的内容即为byte[]
    //socket.broadcast.emit("image", message);
    if(interval > 1){
      io.emit("image", message);
      interval = 0;
    }


  });

  //	 监听到连接断开
  socket.on("disconnect", function () {
    count--;
    console.log(name + "离开了群聊")
    io.emit("disconnection", {
      count: count,
      name: name
    });
  });



});

module.exports = app;

index.js(用户页面的路由)

var express = require('express');
var router = express.Router();

/* GET home page. */
router.get('/', function (req, res) {
    res.render('sys', { title: 'Express' });
  
});

module.exports = router;

sys.ejs（ejs模板引擎）重要的部分写了注释

<!DOCTYPE html>
<html lang="zh-CN">

<head>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <!-- 上述3个meta标签*必须*放在最前面，任何其他内容都*必须*跟随其后！ -->
    <title>1552218</title>

    <!-- Bootstrap -->
    <link href="https://cdn.jsdelivr.net/npm/bootstrap@3.3.7/dist/css
/bootstrap.min.css" rel="stylesheet">
  <style>
    * {
      margin: 0;
      padding: 0;
      box-sizing: border-box;
    }
    body {
      font: 13px Helvetica, Arial;
    }
    form {
      background: #000;
      padding: 3px;
      position: fixed;
      bottom: 0;
      width: 100%;
    }
    form input {
      border: 0;
      padding: 10px;
      width: 90%;
      margin-right: .5%;
    }
    form button {
      width: 9%;
      background: rgb(130, 224, 255);
      border: none;
      padding: 10px;
    }
    #messages {
      list-style-type: none;
      margin: 0;
      padding: 0;
    }
    #messages li {
      padding: 5px 10px;
    }
    #messages li:nth-child(odd) {
      background: #eee;
    }
  </style>
</head>
<script src="https://code.jquery.com/jquery-1.11.1.js"></script>
<script src="js/socket.io.js"></script>
<script>
  //这个函数可以不管
  function encode (input) {
    var keyStr = 
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
    var output = "";
    var chr1, chr2, chr3, enc1, enc2, enc3, enc4;
    var i = 0;

    while (i < input.length) {
        chr1 = input[i++];
        chr2 = i < input.length ? input[i++] : Number.NaN; 
// Not sure if the index 
        chr3 = i < input.length ? input[i++] : Number.NaN; 
// checks are needed here

        enc1 = chr1 >> 2;
        enc2 = ((chr1 & 3) << 4) | (chr2 >> 4);
        enc3 = ((chr2 & 15) << 2) | (chr3 >> 6);
        enc4 = chr3 & 63;

        if (isNaN(chr2)) {
            enc3 = enc4 = 64;
        } else if (isNaN(chr3)) {
            enc4 = 64;
        }
        output += keyStr.charAt(enc1) + keyStr.charAt(enc2) +
                  keyStr.charAt(enc3) + keyStr.charAt(enc4);
    }
    return output;
}


  $(function () {
    var socket = io();//初始化socketio

    //处理image事件
    //这里的处理十分简单，就是将单帧的数据显示出来，也没有什么优化，所以会有点卡
    //更加进阶的方法应该是用视频流来处理，但是我也不知道咋搞
    socket.on('image', function (msg) {
      var arrayBufferView = new Uint8Array(msg);
//首先将数据转换为Uint8Array,一个个的字节
      //console.log(arrayBufferView);
      var blob = new Blob([arrayBufferView], { type: "image/jpeg" });
//存储为Blob对象
      //console.log(blob);
      //var urlCreator = window.URL || window.webkitURL;
      var imageUrl = URL.createObjectURL(blob);//转化为URL
      var img = document.getElementById("screen");//然后就可以用url来显示了
      // var img = document.querySelector("#photo");
      img.src = imageUrl;
//"data:image/jpg;base64,"+encode(arrayBufferView);
    });
  });
</script>

<body>
  <img id="screen" src="">
</body>

</html>

4.  android端代码

package com.example.o0orick.camera;

import android.graphics.Bitmap;
import android.graphics.SurfaceTexture;
import android.hardware.usb.UsbDevice;
import android.os.Bundle;
import android.util.Log;
import android.view.Surface;
import android.view.View;
import android.view.View.OnClickListener;
import android.view.Window;
import android.view.WindowManager;
import android.widget.ImageButton;
import android.widget.ImageView;
import android.widget.Toast;

import com.serenegiant.common.BaseActivity;
import com.serenegiant.usb.CameraDialog;
import com.serenegiant.usb.IFrameCallback;
import com.serenegiant.usb.USBMonitor;
import com.serenegiant.usb.USBMonitor.OnDeviceConnectListener;
import com.serenegiant.usb.USBMonitor.UsbControlBlock;
import com.serenegiant.usb.UVCCamera;
import com.serenegiant.usbcameracommon.UVCCameraHandler;
import com.serenegiant.widget.CameraViewInterface;

import java.io.ByteArrayOutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.ByteBuffer;
import io.socket.client.IO;
import io.socket.client.Socket;
import okio.ByteString;

public final class MainActivity extends BaseActivity implements CameraDialog.CameraDialogParent {
    private static final boolean DEBUG = true;	// TODO set false on release
    private static final String TAG = "MainActivity";

    /**
     * 操作锁
     */
	private final Object mSync = new Object();

    /**
     * set true if you want to record movie using MediaSurfaceEncoder
     * (writing frame data into Surface camera from MediaCodec
     *  by almost same way as USBCameratest2)
     * set false if you want to record movie using MediaVideoEncoder
     */
    private static final boolean USE_SURFACE_ENCODER = false;

    /**
     * preview resolution(width)
     * if your camera does not support specific resolution and mode,
     * {@link UVCCamera#setPreviewSize(int, int, int)} throw exception
     */
    private static final int PREVIEW_WIDTH = 640; // 640
    /**
     * preview resolution(height)
     * if your camera does not support specific resolution and mode,
     * {@link UVCCamera#setPreviewSize(int, int, int)} throw exception
     */
    private static final int PREVIEW_HEIGHT = 480; //480
    /**
     * preview mode
     * if your camera does not support specific resolution and mode,
     * {@link UVCCamera#setPreviewSize(int, int, int)} throw exception
     * 0:YUYV, other:MJPEG
     */
    private static final int PREVIEW_MODE = 0; // YUV

    protected static final int SETTINGS_HIDE_DELAY_MS = 2500;

    /**
     * for accessing USB
     */
    private USBMonitor mUSBMonitor;
    /**
     * Handler to execute camera related methods sequentially on private thread
     */
    private UVCCameraHandler mCameraHandler;
    /**
     * for camera preview display
     */
    private CameraViewInterface mUVCCameraView;
    /**
     * for open&start / stop&close camera preview
     */
    private ImageButton mCameraButton;
    private ImageView mImageView;
    private boolean isScaling = false;
    private boolean isInCapturing = false;

    private int[][] capture_solution = {{640,480}, {800,600},{1024,768}, {1280,1024}};
    private int mCaptureWidth = capture_solution[0][0];
    private int mCaptureHeight = capture_solution[0][1];

    public static final String CHAT_SERVER_URL = "http://192.168.3.12:8081";

    Socket mSocket;

    @Override
    protected void onResume() {
        super.onResume();

    }


    @Override
    protected void onCreate(final Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        Log.v(TAG, "onCreate:");

        onResume();

        getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
        requestWindowFeature(Window.FEATURE_NO_TITLE);
        getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN,
                WindowManager.LayoutParams.FLAG_FULLSCREEN);

        setContentView(R.layout.activity_main);
        mCameraButton = findViewById(R.id.imageButton);
        mCameraButton.setOnClickListener(mOnClickListener);

        mCaptureWidth = capture_solution[0][0];
        mCaptureHeight = capture_solution[0][1];
        bitmap = Bitmap.createBitmap(mCaptureWidth, mCaptureHeight, Bitmap.Config.RGB_565);

        final View view = findViewById(R.id.camera_view);
        mUVCCameraView = (CameraViewInterface)view;
        mUVCCameraView.setAspectRatio(PREVIEW_WIDTH / (float)PREVIEW_HEIGHT);

        try {
            mSocket = IO.socket(CHAT_SERVER_URL);

        } catch (URISyntaxException e) {
            Log.v(TAG, "Socket Failed");
            throw new RuntimeException(e);

        }

		synchronized (mSync) {
	        mUSBMonitor = new USBMonitor(this, mOnDeviceConnectListener);
	        mCameraHandler = UVCCameraHandler.createHandler(this, mUVCCameraView,
	                USE_SURFACE_ENCODER ? 0 : 1, PREVIEW_WIDTH, PREVIEW_HEIGHT, PREVIEW_MODE);
		}
    }

    @Override
    protected void onStart() {
        super.onStart();

        mSocket.connect();
        Log.v(TAG, "onStart:");

		synchronized (mSync) {
        	mUSBMonitor.register();
		}
		if (mUVCCameraView != null) {
  			mUVCCameraView.onResume();
		}
    }

    @Override
    protected void onStop() {
        Log.v(TAG, "onStop:");
        synchronized (mSync) {
    		mCameraHandler.close();	// #close include #stopRecording and #stopPreview
			mUSBMonitor.unregister();
        }
		 if (mUVCCameraView != null)
			mUVCCameraView.onPause();
        super.onStop();
    }

    @Override
    public void onDestroy() {
        Log.v(TAG, "onDestroy:");
        synchronized (mSync) {
            if (mCameraHandler != null) {
                mCameraHandler.setPreviewCallback(null); //zhf
                mCameraHandler.release();
                mCameraHandler = null;
            }
            if (mUSBMonitor != null) {
                mUSBMonitor.destroy();
                mUSBMonitor = null;
            }
        }
        super.onDestroy();
    }

    /**
     * event handler when click camera / capture button
     */
    private final OnClickListener mOnClickListener = new OnClickListener() {
        @Override
        public void onClick(final View view) {
            synchronized (mSync) {
                if ((mCameraHandler != null) && !mCameraHandler.isOpened()) {
                    CameraDialog.showDialog(MainActivity.this);
                } else {
                    mCameraHandler.close();
                }
            }
        }
    };

    private void startPreview() {
		synchronized (mSync) {
			if (mCameraHandler != null) {
                final SurfaceTexture st = mUVCCameraView.getSurfaceTexture();
                /**
                 * 由于surfaceview由另一个线程处理，这里使用消息处理机制
                 * 对Frame进行回调处理
                 */
				mCameraHandler.setPreviewCallback(mIFrameCallback);
                mCameraHandler.startPreview(new Surface(st));
			}
		}
        updateItems();
    }

    private final OnDeviceConnectListener mOnDeviceConnectListener = new OnDeviceConnectListener() {
        @Override
        public void onAttach(final UsbDevice device) {
            Toast.makeText(MainActivity.this, "USB_DEVICE_ATTACHED", Toast.LENGTH_SHORT).show();
        }

        @Override
        public void onConnect(final UsbDevice device, final UsbControlBlock ctrlBlock, final boolean createNew) {
            if (DEBUG) Log.v(TAG, "onConnect:");
            synchronized (mSync) {
                if (mCameraHandler != null) {
	                mCameraHandler.open(ctrlBlock);
	                startPreview();
	                updateItems();
				}
            }
        }

        @Override
        public void onDisconnect(final UsbDevice device, final UsbControlBlock ctrlBlock) {
            if (DEBUG) Log.v(TAG, "onDisconnect:");
            synchronized (mSync) {
                if (mCameraHandler != null) {
                    queueEvent(new Runnable() {
                        @Override
                        public void run() {
                            try{
                                // maybe throw java.lang.IllegalStateException: already released
                                mCameraHandler.setPreviewCallback(null); //zhf
                            }
                            catch(Exception e){
                                e.printStackTrace();
                            }
                            mCameraHandler.close();
                        }
                    }, 0);
				}
            }
        }
        @Override
        public void onDettach(final UsbDevice device) {
            Toast.makeText(MainActivity.this, "USB_DEVICE_DETACHED", Toast.LENGTH_SHORT).show();
        }

        @Override
        public void onCancel(final UsbDevice device) {
        }
    };

    /**
     * to access from CameraDialog
     * @return
     */
    @Override
    public USBMonitor getUSBMonitor() {
		synchronized (mSync) {
			return mUSBMonitor;
		}
	}

    @Override
    public void onDialogResult(boolean canceled) {
        if (DEBUG) Log.v(TAG, "onDialogResult:canceled=" + canceled);
    }

    //================================================================================
    private boolean isActive() {
        return mCameraHandler != null && mCameraHandler.isOpened();
    }

    private boolean checkSupportFlag(final int flag) {
        return mCameraHandler != null && mCameraHandler.checkSupportFlag(flag);
    }

    private int getValue(final int flag) {
        return mCameraHandler != null ? mCameraHandler.getValue(flag) : 0;
    }

    private int setValue(final int flag, final int value) {
        return mCameraHandler != null ? mCameraHandler.setValue(flag, value) : 0;
    }

    private int resetValue(final int flag) {
        return mCameraHandler != null ? mCameraHandler.resetValue(flag) : 0;
    }

    /**
     * 利用Activity.runOnUiThread(Runnable)把更新ui的代码创建在Runnable中，
     * 然后在需要更新ui时，把这个Runnable对象传给Activity.runOnUiThread(Runnable)
     */
    private void updateItems() {
        runOnUiThread(mUpdateItemsOnUITask, 30);
    }

    private final Runnable mUpdateItemsOnUITask = new Runnable() {
        @Override
        public void run() {
            if (isFinishing()) return;
            final int visible_active = isActive() ? View.VISIBLE : View.INVISIBLE;
        }
    };

    // if you need frame data as byte array on Java side, you can use this callback method with UVCCamera#setFrameCallback
    // if you need to create Bitmap in IFrameCallback, please refer following snippet.
    private Bitmap bitmap = null;//Bitmap.createBitmap(640, 480, Bitmap.Config.RGB_565);
    private final Bitmap srcBitmap = Bitmap.createBitmap(PREVIEW_WIDTH, PREVIEW_HEIGHT, Bitmap.Config.RGB_565);
    private String WarnText;

    public static Bitmap getTransformBitmap(Bitmap src) {
        int width = src.getWidth();         // 宽
        int height = src.getHeight();       // 高
        int count = src.getByteCount();     // 获取图片的RGB 565颜色数组总数
        ByteBuffer buffer = ByteBuffer.allocate(count);
        src.copyPixelsToBuffer(buffer);
        byte[] data = buffer.array();       //获取数组
        int sum = width * height;
        int[] pixels = new int[sum];

        for (int i = 0; i < sum; i++) {
            int tmpint = data[i * 2 + 0] + data[i * 2 + 1] * 256;
            int a = 0xff;                           //透明度
            int r = (tmpint & 0xf800) >> 11;        //红
            int g = (tmpint & 0x07e0) >> 5;         //绿
            int b = (tmpint & 0x001f);              //蓝

            r = r << 3;
            g = g << 2;
            b = b << 3;
            pixels[i] = (a << 24) | (r << 16) | (g << 8) | (b);
        }
        Bitmap bmp = Bitmap.createBitmap(width, height,
                Bitmap.Config.ARGB_8888);
        bmp.setPixels(pixels, 0, width, 0, 0, width, height);
        return bmp;    //旋转屏幕
    }

    private void sendBitmap(Bitmap it){

        //Bitmap bmp = getTransformBitmap(it);
        ByteArrayOutputStream buf;
        buf = new ByteArrayOutputStream();
        it.compress(Bitmap.CompressFormat.JPEG, 20, buf);
        //byte[] b = buf.toByteArray();
        mSocket.emit("image", buf.toByteArray());

        Log.v(TAG, "In this module");
    }

    private final IFrameCallback mIFrameCallback = new IFrameCallback() {
        @Override
        public void onFrame(final ByteBuffer frame) {
            frame.clear();
            if(!isActive() || isInCapturing){
                return;
            }
            if(bitmap == null){
                Toast.makeText(MainActivity.this, "错误：Bitmap为空", Toast.LENGTH_SHORT).show();
                return;
            }
            /**
             * 这里进行opencv操作
             * srcBitmap:源
             * bitmap:处理后
             * 转发bitmap数据
             */
            synchronized (bitmap) {
                srcBitmap.copyPixelsFromBuffer(frame);
                WarnText = "";

                if(bitmap.getWidth() != mCaptureWidth || bitmap.getHeight() != mCaptureHeight){
                    bitmap = Bitmap.createBitmap(mCaptureWidth, mCaptureHeight, Bitmap.Config.RGB_565);
                }

                sendBitmap(srcBitmap);
            }
            mImageView.post(mUpdateImageTask);
        }
    };

    private final Runnable mUpdateImageTask = new Runnable() {
        @Override
        public void run() {
            synchronized (bitmap) {
                mImageView.setImageBitmap(bitmap);

            }
        }
    };

}

---

END