OpenCV读取ESP32CAM的视频流

---

title: OpenCV读取ESP32CAM的视频流

直接看代码

这是博主的博文建议直接去看

#include <esp32cam.h>
#include <WebServer.h>
#include <WiFi.h>

const char* WIFI_SSID = "HONOR V20";  // 改成自己的wifi名称
const char* WIFI_PASS = "*****";  // 改成自己的wifi密码

WebServer server(80);

static auto loRes = esp32cam::Resolution::find(320, 240);
static auto hiRes = esp32cam::Resolution::find(800, 600);

void handleBmp()
{
  if (!esp32cam::Camera.changeResolution(loRes)) {
    Serial.println("SET-LO-RES FAIL");
  }

  auto frame = esp32cam::capture();
  if (frame == nullptr) {
    Serial.println("CAPTURE FAIL");
    server.send(503, "", "");
    return;
  }
  Serial.printf("CAPTURE OK %dx%d %db\n", frame->getWidth(), frame->getHeight(),
                static_cast<int>(frame->size()));

  if (!frame->toBmp()) {
    Serial.println("CONVERT FAIL");
    server.send(503, "", "");
    return;
  }
  Serial.printf("CONVERT OK %dx%d %db\n", frame->getWidth(), frame->getHeight(),
                static_cast<int>(frame->size()));

  server.setContentLength(frame->size());
  server.send(200, "image/bmp");
  WiFiClient client = server.client();
  frame->writeTo(client);
}

void serveJpg()
{
  auto frame = esp32cam::capture();
  if (frame == nullptr) {
    Serial.println("CAPTURE FAIL");
    server.send(503, "", "");
    return;
  }
  Serial.printf("CAPTURE OK %dx%d %db\n", frame->getWidth(), frame->getHeight(),
                static_cast<int>(frame->size()));

  server.setContentLength(frame->size());
  server.send(200, "image/jpeg");
  WiFiClient client = server.client();
  frame->writeTo(client);
}

void handleJpgLo()
{
  if (!esp32cam::Camera.changeResolution(loRes)) {
    Serial.println("SET-LO-RES FAIL");
  }
  serveJpg();
}

void handleJpgHi()
{
  if (!esp32cam::Camera.changeResolution(hiRes)) {
    Serial.println("SET-HI-RES FAIL");
  }
  serveJpg();
}

void handleJpg()
{
  server.sendHeader("Location", "/cam-hi.jpg");
  server.send(302, "", "");
}

void handleMjpeg()
{
  if (!esp32cam::Camera.changeResolution(hiRes)) {
    Serial.println("SET-HI-RES FAIL");
  }

  Serial.println("STREAM BEGIN");
  WiFiClient client = server.client();
  auto startTime = millis();
  int res = esp32cam::Camera.streamMjpeg(client);
  if (res <= 0) {
    Serial.printf("STREAM ERROR %d\n", res);
    return;
  }
  auto duration = millis() - startTime;
  Serial.printf("STREAM END %dfrm %0.2ffps\n", res, 1000.0 * res / duration);
}

void setup()
{
  Serial.begin(115200);
  Serial.println();

  {
    using namespace esp32cam;
    Config cfg;
    cfg.setPins(pins::AiThinker);
    cfg.setResolution(hiRes);
    cfg.setBufferCount(2);
    cfg.setJpeg(80);

    bool ok = Camera.begin(cfg);
    Serial.println(ok ? "CAMERA OK" : "CAMERA FAIL");
  }

  WiFi.persistent(false);
  WiFi.mode(WIFI_STA);
  WiFi.begin(WIFI_SSID, WIFI_PASS);
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
  }

  Serial.print("http://");
  Serial.println(WiFi.localIP());
  Serial.println("  /cam.bmp");
  Serial.println("  /cam-lo.jpg");
  Serial.println("  /cam-hi.jpg");
  Serial.println("  /cam.mjpeg");

  server.on("/cam.bmp", handleBmp);
  server.on("/cam-lo.jpg", handleJpgLo);
  server.on("/cam-hi.jpg", handleJpgHi);
  server.on("/cam.jpg", handleJpg);
  server.on("/cam.mjpeg", handleMjpeg);

  server.begin();
}

void loop()
{
  server.handleClient();
}

结果

还是卡的一比，和rstp流解析的一模一样，我都服了，这回真的没办法了只能想办法弄台配置高的电脑

换了一个思路

将视频改为图片定时读取识别也能满足基本的使用要求
使用框架大致为:
esp32cam 读取图片 —> wifi —> MQTT云服务器 <— python获取图片并识别返回结果

esp32端代码(C++)

#include "WiFi.h"
#include "esp_camera.h"
#include "esp_timer.h"
#include "img_converters.h"
#include "Arduino.h"
#include "soc/soc.h"           // Disable brownour problems
#include "soc/rtc_cntl_reg.h"  // Disable brownour problems
#include "driver/rtc_io.h"
//#include "StringArray.h"

#include <PubSubClient.h>
#include <base64.h>
#include <libb64/cencode.h>


//esp32-cam针脚定义 拍照相关
constexpr int kCameraPin_PWDN   =  32;
constexpr int kCameraPin_RESET  =  -1;  // NC
constexpr int kCameraPin_XCLK   =   0;
constexpr int kCameraPin_SIOD   =  26;
constexpr int kCameraPin_SIOC   =  27;
constexpr int kCameraPin_Y9     =  35;
constexpr int kCameraPin_Y8     =  34;
constexpr int kCameraPin_Y7     =  39;
constexpr int kCameraPin_Y6     =  36;
constexpr int kCameraPin_Y5     =  21;
constexpr int kCameraPin_Y4     =  19;
constexpr int kCameraPin_Y3     =  18;
constexpr int kCameraPin_Y2     =   5;
constexpr int kCameraPin_VSYNC  =  25;
constexpr int kCameraPin_HREF   =  23;
constexpr int kCameraPin_PCLK   =  22;

// WiFi
const char *ssid = "HONOR V20"; // Enter your WiFi name
const char *password = " ***** ";  // Enter WiFi password

// MQTT Broker
const char *mqtt_broker = " ***** "; 
const char *mqtt_TopicName = "*****";
const char *mqtt_username = "*****";
const char *mqtt_password = " ***** ";
const int mqtt_port = *****;

//设置相机帧的分辨率大小
// framesize_t resolution_ = FRAMESIZE_240X240;


//定义延时时间1000=1s
#define SLEEP_DELAY 60000 //延迟60s
#define FILE_PHOTO "/photo.jpg"

// OV2640 相机模组的针脚定义
#define PWDN_GPIO_NUM     32
#define RESET_GPIO_NUM    -1
#define XCLK_GPIO_NUM      0
#define SIOD_GPIO_NUM     26
#define SIOC_GPIO_NUM     27
#define Y9_GPIO_NUM       35
#define Y8_GPIO_NUM       34
#define Y7_GPIO_NUM       39
#define Y6_GPIO_NUM       36
#define Y5_GPIO_NUM       21
#define Y4_GPIO_NUM       19
#define Y3_GPIO_NUM       18
#define Y2_GPIO_NUM        5
#define VSYNC_GPIO_NUM    25
#define HREF_GPIO_NUM     23
#define PCLK_GPIO_NUM     22


// 选择板子型号
//#define CAMERA_MODEL_WROVER_KIT
//#define CAMERA_MODEL_ESP_EYE
//#define CAMERA_MODEL_M5STACK_PSRAM
//#define CAMERA_MODEL_M5STACK_WIDE
#define CAMERA_MODEL_AI_THINKER


//#define ESP32_CLIENT_ID = WiFi.macAddress()
//const char* esp_client_id = WiFi.macAddress()
WiFiClient mqttClient;
PubSubClient client(mqttClient);

// const int LED_BUILTIN = 4;
// boolean shotflag = false;

void setup_camera() {

     // OV2640 camera module
     camera_config_t config;
    config.pin_pwdn     = kCameraPin_PWDN;
    config.pin_reset    = kCameraPin_RESET;
    config.pin_xclk     = kCameraPin_XCLK;
    config.pin_sscb_sda = kCameraPin_SIOD;
    config.pin_sscb_scl = kCameraPin_SIOC;
    config.pin_d7       = kCameraPin_Y9;
    config.pin_d6       = kCameraPin_Y8;
    config.pin_d5       = kCameraPin_Y7;
    config.pin_d4       = kCameraPin_Y6;
    config.pin_d3       = kCameraPin_Y5;
    config.pin_d2       = kCameraPin_Y4;
    config.pin_d1       = kCameraPin_Y3;
    config.pin_d0       = kCameraPin_Y2;
    config.pin_vsync    = kCameraPin_VSYNC;
    config.pin_href     = kCameraPin_HREF;
    config.pin_pclk     = kCameraPin_PCLK;
    config.xclk_freq_hz = 20000000;
    config.ledc_timer   = LEDC_TIMER_0;
    config.ledc_channel = LEDC_CHANNEL_0;
    config.pixel_format = PIXFORMAT_JPEG;
    config.frame_size   = FRAMESIZE_VGA;    //640x480 
    config.jpeg_quality = 10;
    config.fb_count     = 1;

  esp_err_t err = esp_camera_init(&config);
  Serial.printf("esp_camera_init: 0x%x\n", err);

  // sensor_t *s = esp_camera_sensor_get();
  // s->set_framesize(s, FRAMESIZE_QVGA);   
  }

String msg;
int timeCount = 0;
void getimg(){//拍照分段发送到mqtt

    camera_fb_t *fb = esp_camera_fb_get();
    if (fb){
        Serial.printf("width: %d, height: %d, buf: 0x%x, len: %d\n", fb->width, fb->height, fb->buf, fb->len);
        char data[4104];
        //client.publish(mqtt_TopicName, "0");
        for (int i = 0; i < fb->len; i++){

            sprintf(data, "%02X", *((fb->buf + i)));
            msg += data;
            if (msg.length() == 4096){
                timeCount += 1;
                client.beginPublish(mqtt_TopicName, msg.length(), 0);
                client.print(msg);
                client.endPublish();
                msg = "";
            }
        }
        if (msg.length() > 0){
            client.beginPublish(mqtt_TopicName, msg.length(), 0);
            client.print(msg);
            client.endPublish();
            msg = "";
        }
        client.publish(mqtt_TopicName, "1");
        timeCount = 0;
        esp_camera_fb_return(fb);
    }
}

void setup_wifi() {
  delay(10);
  // 连接WIFI
  Serial.println();
  Serial.print("Connecting to ");
  Serial.println(ssid);

  WiFi.begin(ssid, password);

  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }

  Serial.println("");
  Serial.println("WiFi connected");
  Serial.print("IP address : ");
  Serial.println(WiFi.localIP());
}

//MQTT重连
void reconnect() {
  while (!client.connected()) {
    Serial.print("Attempting MQTT connection...");
    if (client.connect("ESP32CAMClient", mqtt_username, mqtt_password)) {
      Serial.println("connected");
    } else {
      Serial.print("failed, rc=");
      Serial.print(client.state());
      Serial.println(" try again in 5 seconds");
      delay(5000);
    }
  }
}

// 回调函数
// void callback(char *topic, byte *payload, unsigned int length)
// {
//     // Serial.print("Message arrived [");
//     // Serial.print(topic); // 打印主题信息
//     // Serial.print("] ");
//     // for (int i = 0; i < length; i++)
//     // {
//     //     Serial.print((char)payload[i]); // 打印主题内容
//     // }
//     // Serial.println();
//     if ((char)payload[0] == '1')
//     {
//         shotflag = true;
//     }
//     if ((char)payload[0] == '0')
//     {
//         shotflag = false;
//     }
// }


void setup() {
  Serial.begin(115200);
  Serial.setDebugOutput(true);
  Serial.println();
  setup_camera(); //设置相机
  setup_wifi();		//连接WIFI
  client.setServer(mqtt_broker, mqtt_port); //连接MQTT Broker
  if (client.connect("ESP32CAMClient", mqtt_username, mqtt_password)) {
      Serial.println("mqtt connected");
    } 
  // // publish and subscribe
  // client.publish(mqtt_TopicName, "Hi EMQX I'm ESP32 ^^");
  // client.subscribe(mqtt_TopicName);
}

void loop() {
  getimg();

  if (!client.connected()) {
    reconnect();
  }

  delay(1000);
}

上位机端代码（Python）

"""
MAQTT协议
ESP32抓拍推送至云服务器
PC端订阅处理抓拍的图片
"""
import binascii
import os
import random
import cv2
import numpy as np
import torch
from paho.mqtt import client as mqtt_client

# --------------MQTT1初始化-------------------#
broker = '*****'
port = *****
topic = "*****" 
topic1 = "*****"  
# generate client ID with pub prefix randomly
client_id = f'python-mqtt-{random.randint(0, 100)}'

# ---------------YOLO初始化------------------#
# 加载训练模型
model = torch.hub.load('./yolov5', 'custom', path='./weight/yolov5s.pt', source='local')
# 设置阈值
model.conf = 0.52  # confidence threshold (0-1)
model.iou = 0.45  # NMS IoU threshold (0-1)


class GlobalValue:
    data = [""]
    temp = 0


def connect_mqtt() -> mqtt_client:
    def on_connect(client, userdata, flags, rc):
        if rc == 0:
            print("Connected to MQTT Broker!")
        else:
            print("Failed to connect, return code %d\n", rc)

    client = mqtt_client.Client(client_id)
    client.on_connect = on_connect
    client.connect(broker, port)
    return client


# 订阅
def subscribe(client: mqtt_client):
    def on_message(client, userdata, msg):
        """
        回调函数
        """
        print(f"Received `{msg.payload.decode()}` from `{msg.topic}` topic")
        GlobalValue.data.append(str(msg.payload.decode()))  # 获取十六进制图片数据

    client.subscribe(topic)
    client.on_message = on_message


def run():
    client = connect_mqtt()
    subscribe(client)
    imgcount = 0
    accuracy = 0
    while True:
        client.loop_start()
        # 判断一张图片是否发完
        if (GlobalValue.data[len(GlobalValue.data) - 1]) == "1":
            # print("Reached EOF")
            data1 = "".join(GlobalValue.data[0:(len(GlobalValue.data) - 1)])
            GlobalValue.data = [""]
            data1 = binascii.a2b_hex(data1)     # 解析十六进制图片流
            data1 = np.frombuffer(data1, dtype=np.uint8)    # array格式存在buffer中
            img = cv2.imdecode(data1, 1)    # Reads an image from a buffer in memory
            # 保存图片(经测试可使用)
            save_img_path = "SaveImg/image" + str(imgcount) + ".jpg"
            with open(save_img_path, "wb") as image_file:
                image_file.write(data1)
            cv2.imshow("Door", img)
            # 使用YOLO处理图片
            result, temp = yolo_detect(img)
            cv2.imshow('result', result)
            # ------------------反馈信息给esp32屏幕 Start-------------- #
            if temp > 0:
                accuracy += 1
                msg = "on"
                client.publish(topic1, msg)
            else:
                msg = "off"
                client.publish(topic1, msg)
            # ------------------反馈信息给esp32屏幕 End-------------- #
            # 保存处理后的图片 
            label = "ResultImg/image" + str(imgcount) + ".jpg"
            cv2.imwrite(label, result)
            cv2.waitKey(1)
            if imgcount >= 100:  # 设定抓拍100张后结束
                print(f'识别正确率为：{accuracy/100}')
                exit(0)
            else:
                imgcount += 1


def yolo_detect(img):
    """
    yolo处理图片
    """
    # Inference
    results = model(img)
    # Results
    # 显示相关结果信息，如：图片信息、识别速度...
    results.print()  # or .show(), .save(), .crop(), .pandas(), etc.
    temp = results.xyxy[0]  # 识别结果用tensor保存数据，包含标签、坐标范围、IOU
    print(temp)
    # GPU 转 CPU
    temp = temp.cpu()
    # tensor 转 array
    temp = temp.numpy()  # tensor --> array格式
    # 显示
    # 框出所识别的物体
    if len(temp) > 0:
        cv2.rectangle(img, (int(temp[0, 0]), int(temp[0, 1])), (int(temp[0, 2]), int(temp[0, 3])),
                      (0, 0, 255), 3)  # 框出识别物体

    return img, len(temp)


if __name__ == '__main__':
    run()