将esp32-s3-eye做为USB网络摄像头（UVC设备）

咚咚锵咚咚锵

已于 2025-03-21 08:49:45 修改

阅读量3.3k

点赞数 5

分类专栏：模型落地文章标签：网络 python 深度学习

于 2024-09-10 17:28:35 首次发布

本文链接：https://blog.csdn.net/weixin_45977690/article/details/142103736

版权

模型落地专栏收录该内容

13 篇文章

订阅专栏

硬件要求

官方默认的USB WebCam config就是乐鑫带摄像头OV2604的esp32-s3-eye，其他的开发板可以参考官方网站进行配置。

摄像头要求

参考esp32-camera查看摄像头是否支持JPEG压缩；
使用idf.py menuconfig，USB WebCam config配置帧分辨率、帧速率和图像质量；
通过USB WebCam config -> UVC transfer mode，用户可以切换到Bulk模式来获得差不多Isochronous两倍的吞吐量。

编译和烧录

IDF版本测试了v5.0.4和v5.3两个版本，5.3的直接按流程烧录会报下方错误，5.0.4的没问题。esp32s3-eye开发板进入下载模式的操作：按住BOOT键，短暂按下RESET键，松开RESET键，继续按住BOOT键1秒后松开。[to 自己：使用自己的开发板和乐鑫官方开发板时使用记得查看~/esp-iot-solution/examples/usb/device/usb_webcam/main/camera_pin.h的摄像头引脚设置]
在这里插入图片描述

git clone https://github.com/espressif/esp-iot-solution.git
cd ~/esp-iot-solution/examples/usb/device/usb_webcam/
idf.py set-target esp32s3
idf.py build flash moniotr

官方的示例输出

I (0) cpu_start: Starting scheduler on APP CPU.
I (533) spiram: Reserving pool of 32K of internal memory for DMA/internal allocations
I (534) usb_webcam: Selected Camera Board ESP-S3-EYE
I (534) usb_webcam: Format List
I (534) usb_webcam:     Format(1) = MJPEG
I (534) usb_webcam: Frame List
I (535) usb_webcam:     Frame(1) = 1280 * 720 @15fps, Quality = 14
I (535) usb_webcam:     Frame(2) = 640 * 480 @15fps, Quality = 14
I (535) usb_webcam:     Frame(3) = 480 * 320 @30fps, Quality = 10
I (869) usb_webcam: Mount
I (2099) usb_webcam: Suspend
I (9343) usb_webcam: bFrameIndex: 1
I (9343) usb_webcam: dwFrameInterval: 666666
I (9343) s3 ll_cam: DMA Channel=4
I (9344) cam_hal: cam init ok
I (9344) sccb: pin_sda 4 pin_scl 5
I (9344) sccb: sccb_i2c_port=1
I (9354) camera: Detected camera at address=0x30
I (9357) camera: Detected OV2640 camera
I (9357) camera: Camera PID=0x26 VER=0x42 MIDL=0x7f MIDH=0xa2
I (9434) cam_hal: buffer_size: 16384, half_buffer_size: 1024, node_buffer_size: 1024, node_cnt: 16, total_cnt: 180
I (9435) cam_hal: Allocating 184320 Byte frame buffer in PSRAM
I (9435) cam_hal: Allocating 184320 Byte frame buffer in PSRAM
I (9436) cam_hal: cam config ok
I (9436) ov2640: Set PLL: clk_2x: 0, clk_div: 0, pclk_auto: 0, pclk_div: 12

实际上烧录结果如下，应该是烧录后USB口应该被占用了所以没有打印输出。
在这里插入图片描述

电脑端查看摄像头

现在电脑就可以直接访问摄像头了，使用python运行下面代码，在打开摄像头的时候LCD上的动画的眼睛会一直眨巴眨巴。

import cv2

# 打开摄像头，参数0表示第一个摄像头，如果有多个摄像头可以尝试1, 2, 3等
cap = cv2.VideoCapture(0)

if not cap.isOpened():
    print("无法打开摄像头")
    exit()

while True:
    # 读取摄像头的一帧
    ret, frame = cap.read()
    
    if not ret:
        print("无法接收帧 (stream end?). Exiting ...")
        break
    
    # 显示这一帧
    cv2.imshow('Image', frame)
    
    # 按下'q'键退出循环
    if cv2.waitKey(1) == ord('q'):
        break

# 释放摄像头并关闭所有OpenCV窗口
cap.release()
cv2.destroyAllWindows()

在这里插入图片描述

---------------------------------------------------------------------------------------------------------------

不支持MJPEG格式的摄像头的操作

比如我想使用GC0308的摄像头，发现它自身是不支持输出JPEG格式的图像，那么就需要在摄像头输出后进行一个格式的转换，我使用RGB565转MJPEG，代码很粗糙。
在这里插入图片描述
只需要在usb_webcam_main.c函数中增加一些处理
（1）在最前面增加一个宏定义来说明使用的摄像头是否支持MJPEG输出格式：

#define CAMERA_MJPEG_SUPPORT false

（2）GC0308的摄像头成像是倒立的，将其摆正

    if (s->id.PID == OV3660_PID || s->id.PID == OV2640_PID) {
        s->set_vflip(s, 1); // flip it back
    } else if (s->id.PID == GC0308_PID) {
        s->set_hmirror(s, 0);
        s->set_vflip(s, 1); 
    } else if (s->id.PID == GC032A_PID) {
        s->set_vflip(s, 1);
    }

（3）增加对RGB565格式的支持

    if (ESP_OK == ret && PIXFORMAT_JPEG == pixel_format && s_info->support_jpeg == true && CAMERA_MJPEG_SUPPORT == true) {
        ESP_LOGI(TAG, "JPEG format is supported");
        cur_xclk_freq_hz = xclk_freq_hz;
        cur_pixel_format = pixel_format;
        cur_frame_size = frame_size;
        cur_jpeg_quality = jpeg_quality;
        cur_fb_count = fb_count;
        inited = true;
    }  else if (ESP_OK == ret && PIXFORMAT_RGB565 == pixel_format && CAMERA_MJPEG_SUPPORT == false) {
        ESP_LOGI(TAG, "RGB565 format is supported");
        cur_xclk_freq_hz = xclk_freq_hz;
        cur_pixel_format = pixel_format;
        cur_frame_size = frame_size;
        cur_jpeg_quality = jpeg_quality;
        cur_fb_count = fb_count;
        inited = true;
    }
    else {
        ESP_LOGE(TAG, "Neither JPEG nor RGB565 formats are supported");
        return ESP_ERR_NOT_SUPPORTED;
    }

    esp_err_t ret;
    if (CAMERA_MJPEG_SUPPORT)
    {
        ret = camera_init(CAMERA_XCLK_FREQ, PIXFORMAT_JPEG, frame_size, jpeg_quality, CAMERA_FB_COUNT);
    } else {
        ret = camera_init(CAMERA_XCLK_FREQ, PIXFORMAT_RGB565, frame_size, jpeg_quality, CAMERA_FB_COUNT);
    }

（4）如果为RGB565格式的，需要进行转换，可直接调用esp-camera组件中的转换接口frame2jpg

    if (s_fb.cam_fb_p->format == PIXFORMAT_JPEG) {
        printf("frame is jpeg format\n");
        s_fb.uvc_fb.buf = s_fb.cam_fb_p->buf;
        s_fb.uvc_fb.len = s_fb.cam_fb_p->len;
        s_fb.uvc_fb.format = s_fb.cam_fb_p->format;
    } else if (frame2jpg(s_fb.cam_fb_p, 60, &s_fb.uvc_fb.buf, &s_fb.uvc_fb.len)) {
        s_fb.uvc_fb.format = PIXFORMAT_JPEG;
        printf("JPEG compression success, JPEG: %d bytes\n", s_fb.uvc_fb.len);
    } else {
        printf("JPEG compression failed\n");
        return NULL;
    }

（5）frame2jpg函数中有申请内存，所以在一帧结束后需要将内存释放掉

static void camera_fb_return_cb(uvc_fb_t *fb, void *cb_ctx)
{
    (void)cb_ctx;
    assert(fb == &s_fb.uvc_fb);
    esp_camera_fb_return(s_fb.cam_fb_p);
    #if !CAMERA_MJPEG_SUPPORT
        free(fb->buf);
    #endif
}

全部代码如下：

/*
 * SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
 *
 * SPDX-License-Identifier: Apache-2.0
 */

#include <stdlib.h>
#include <string.h>
#include <inttypes.h>
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_timer.h"
#include "esp_log.h"
#include "camera_pin.h"
#include "esp_camera.h"
#include "usb_device_uvc.h"
#include "uvc_frame_config.h"
#if CONFIG_CAMERA_MODULE_ESP_S3_EYE
#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 0, 0)
#include "freertos/event_groups.h"
#include "bsp/esp-bsp.h"
#include "show_eyes.h"
static EventGroupHandle_t s_event_group = NULL;
#define EYES_CLOSE_BIT BIT0
#define EYES_OPEN_BIT  BIT1
#else
#pragma message("ESP-S3-EYE lcd animation not supported in ESP-IDF < v5.0")
#endif
#endif

static const char *TAG = "usb_webcam";

#define CAMERA_XCLK_FREQ           CONFIG_CAMERA_XCLK_FREQ
#define CAMERA_FB_COUNT            2
#define CAMERA_MJPEG_SUPPORT false

#if CONFIG_IDF_TARGET_ESP32S3
#define UVC_MAX_FRAMESIZE_SIZE     (75*1024)
#else
#define UVC_MAX_FRAMESIZE_SIZE     (60*1024)
#endif

typedef struct {
    camera_fb_t *cam_fb_p;
    uvc_fb_t uvc_fb;
} fb_t;

static fb_t s_fb;

static esp_err_t camera_init(uint32_t xclk_freq_hz, pixformat_t pixel_format, framesize_t frame_size, int jpeg_quality, uint8_t fb_count)
{
    static bool inited = false;
    static uint32_t cur_xclk_freq_hz = 0;
    static pixformat_t cur_pixel_format = 0;
    static framesize_t cur_frame_size = 0;
    static int cur_jpeg_quality = 0;
    static uint8_t cur_fb_count = 0;

    if ((inited && cur_xclk_freq_hz == xclk_freq_hz && cur_pixel_format == pixel_format
            && cur_frame_size == frame_size && cur_fb_count == fb_count && cur_jpeg_quality == jpeg_quality)) {
        ESP_LOGD(TAG, "camera already inited");
        return ESP_OK;
    } else if (inited) {
        esp_camera_return_all();
        esp_camera_deinit();
        inited = false;
        ESP_LOGI(TAG, "camera RESTART");
    }

    camera_config_t camera_config = {
        .pin_pwdn = CAMERA_PIN_PWDN,
        .pin_reset = CAMERA_PIN_RESET,
        .pin_xclk = CAMERA_PIN_XCLK,
        .pin_sscb_sda = CAMERA_PIN_SIOD,
        .pin_sscb_scl = CAMERA_PIN_SIOC,

        .pin_d7 = CAMERA_PIN_D7,
        .pin_d6 = CAMERA_PIN_D6,
        .pin_d5 = CAMERA_PIN_D5,
        .pin_d4 = CAMERA_PIN_D4,
        .pin_d3 = CAMERA_PIN_D3,
        .pin_d2 = CAMERA_PIN_D2,
        .pin_d1 = CAMERA_PIN_D1,
        .pin_d0 = CAMERA_PIN_D0,
        .pin_vsync = CAMERA_PIN_VSYNC,
        .pin_href = CAMERA_PIN_HREF,
        .pin_pclk = CAMERA_PIN_PCLK,

        .xclk_freq_hz = xclk_freq_hz,
        .ledc_timer = LEDC_TIMER_0,
        .ledc_channel = LEDC_CHANNEL_0,

        .pixel_format = pixel_format,
        .frame_size = frame_size,

        .jpeg_quality = jpeg_quality,
        .fb_count = fb_count,
        .grab_mode = CAMERA_GRAB_WHEN_EMPTY,
        .fb_location = CAMERA_FB_IN_PSRAM
    };

    // initialize the camera sensor
    esp_err_t ret = esp_camera_init(&camera_config);
    if (ret != ESP_OK) {
        return ret;
    }

    // Get the sensor object, and then use some of its functions to adjust the parameters when taking a photo.
    // Note: Do not call functions that set resolution, set picture format and PLL clock,
    // If you need to reset the appeal parameters, please reinitialize the sensor.
    sensor_t *s = esp_camera_sensor_get();
    s->set_vflip(s, 1); // flip it back
    // initial sensors are flipped vertically and colors are a bit saturated
    if (s->id.PID == OV3660_PID) {
        s->set_brightness(s, 1); // up the blightness just a bit
        s->set_saturation(s, -2); // lower the saturation
    }

    if (s->id.PID == OV3660_PID || s->id.PID == OV2640_PID) {
        s->set_vflip(s, 1); // flip it back
    } else if (s->id.PID == GC0308_PID) {
        s->set_hmirror(s, 0);
        s->set_vflip(s, 1);
    } else if (s->id.PID == GC032A_PID) {
        s->set_vflip(s, 1);
    }

    // Get the basic information of the sensor.
    camera_sensor_info_t *s_info = esp_camera_sensor_get_info(&(s->id));

    if (ESP_OK == ret && PIXFORMAT_JPEG == pixel_format && s_info->support_jpeg == true && CAMERA_MJPEG_SUPPORT == true) {
        ESP_LOGI(TAG, "JPEG format is supported");
        cur_xclk_freq_hz = xclk_freq_hz;
        cur_pixel_format = pixel_format;
        cur_frame_size = frame_size;
        cur_jpeg_quality = jpeg_quality;
        cur_fb_count = fb_count;
        inited = true;
    }  else if (ESP_OK == ret && PIXFORMAT_RGB565 == pixel_format && CAMERA_MJPEG_SUPPORT == false) {
        ESP_LOGI(TAG, "RGB565 format is supported");
        cur_xclk_freq_hz = xclk_freq_hz;
        cur_pixel_format = pixel_format;
        cur_frame_size = frame_size;
        cur_jpeg_quality = jpeg_quality;
        cur_fb_count = fb_count;
        inited = true;
    }
    else {
        ESP_LOGE(TAG, "Neither JPEG nor RGB565 formats are supported");
        return ESP_ERR_NOT_SUPPORTED;
    }

    return ret;
}

static void camera_stop_cb(void *cb_ctx)
{
    (void)cb_ctx;
#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 0, 0)
#if CONFIG_CAMERA_MODULE_ESP_S3_EYE
    xEventGroupSetBits(s_event_group, EYES_CLOSE_BIT);
#endif
#endif
    ESP_LOGI(TAG, "Camera Stop");
}

static esp_err_t camera_start_cb(uvc_format_t format, int width, int height, int rate, void *cb_ctx)
{
    (void)cb_ctx;
    ESP_LOGI(TAG, "Camera Start");
    ESP_LOGI(TAG, "Format: %d, width: %d, height: %d, rate: %d", format, width, height, rate);
    framesize_t frame_size = FRAMESIZE_QVGA;
    int jpeg_quality = 14;

    if (format != UVC_FORMAT_JPEG) {
        ESP_LOGE(TAG, "Only support MJPEG format");
        return ESP_ERR_NOT_SUPPORTED;
    }

    if (width == 320 && height == 240) {
        frame_size = FRAMESIZE_QVGA;
        jpeg_quality = 10;
    } else if (width == 480 && height == 320) {
        frame_size = FRAMESIZE_HVGA;
        jpeg_quality = 10;
    } else if (width == 640 && height == 480) {
        frame_size = FRAMESIZE_VGA;
        jpeg_quality = 12;
    } else if (width == 800 && height == 600) {
        frame_size = FRAMESIZE_SVGA;
        jpeg_quality = 14;
    } else if (width == 1280 && height == 720) {
        frame_size = FRAMESIZE_HD;
        jpeg_quality = 16;
    } else if (width == 1920 && height == 1080) {
        frame_size = FRAMESIZE_FHD;
        jpeg_quality = 16;
    } else {
        ESP_LOGE(TAG, "Unsupported frame size %dx%d", width, height);
        return ESP_ERR_NOT_SUPPORTED;
    }

    esp_err_t ret;
    if (CAMERA_MJPEG_SUPPORT)
    {
        ret = camera_init(CAMERA_XCLK_FREQ, PIXFORMAT_JPEG, frame_size, jpeg_quality, CAMERA_FB_COUNT);
    } else {
        ret = camera_init(CAMERA_XCLK_FREQ, PIXFORMAT_RGB565, frame_size, jpeg_quality, CAMERA_FB_COUNT);
    }
    
    if (ret != ESP_OK) {
        ESP_LOGE(TAG, "camera init failed");
        return ret;
    }

#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 0, 0)
#if CONFIG_CAMERA_MODULE_ESP_S3_EYE
    xEventGroupSetBits(s_event_group, EYES_OPEN_BIT);
#endif
#endif
    return ESP_OK;
}

static uvc_fb_t* camera_fb_get_cb(void *cb_ctx)
{
    (void)cb_ctx;
    s_fb.cam_fb_p = esp_camera_fb_get();
    if (!s_fb.cam_fb_p) {
        return NULL;
    }
    if (s_fb.cam_fb_p->format == PIXFORMAT_JPEG) {
        printf("frame is jpeg format\n");
        s_fb.uvc_fb.buf = s_fb.cam_fb_p->buf;
        s_fb.uvc_fb.len = s_fb.cam_fb_p->len;
        s_fb.uvc_fb.format = s_fb.cam_fb_p->format;
    } else if (frame2jpg(s_fb.cam_fb_p, 60, &s_fb.uvc_fb.buf, &s_fb.uvc_fb.len)) {
        s_fb.uvc_fb.format = PIXFORMAT_JPEG;
        printf("JPEG compression success, JPEG: %d bytes\n", s_fb.uvc_fb.len);
    } else {
        printf("JPEG compression failed\n");
        return NULL;
    }

    s_fb.uvc_fb.width = s_fb.cam_fb_p->width;
    s_fb.uvc_fb.height = s_fb.cam_fb_p->height;
    s_fb.uvc_fb.timestamp = s_fb.cam_fb_p->timestamp;

    if (s_fb.uvc_fb.len > UVC_MAX_FRAMESIZE_SIZE) {
        ESP_LOGE(TAG, "Frame size %d is larger than max frame size %d", s_fb.uvc_fb.len, UVC_MAX_FRAMESIZE_SIZE);
        esp_camera_fb_return(s_fb.cam_fb_p);
        return NULL;
    }
    return &s_fb.uvc_fb;
}

static void camera_fb_return_cb(uvc_fb_t *fb, void *cb_ctx)
{
    (void)cb_ctx;
    assert(fb == &s_fb.uvc_fb);
    esp_camera_fb_return(s_fb.cam_fb_p);
    #if !CAMERA_MJPEG_SUPPORT
        free(fb->buf);
    #endif
}

void app_main(void)
{
    // if using esp-s3-eye board, show the GUI
#if CONFIG_CAMERA_MODULE_ESP_S3_EYE
#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 0, 0)
    bsp_display_start();
    bsp_display_backlight_on();
    lv_obj_t* img = eyes_init();
    s_event_group = xEventGroupCreate();
    xEventGroupSetBits(s_event_group, EYES_CLOSE_BIT);
#else
    ESP_LOGW(TAG, "ESP-S3-EYE lcd animation not supported in ESP-IDF < v5.0");
#endif
#endif
    ESP_LOGI(TAG, "Selected Camera Board %s", CAMERA_MODULE_NAME);
    uint8_t *uvc_buffer = (uint8_t *)malloc(UVC_MAX_FRAMESIZE_SIZE);
    if (uvc_buffer == NULL) {
        ESP_LOGE(TAG, "malloc frame buffer fail");
        return;
    }

    uvc_device_config_t config = {
        .uvc_buffer = uvc_buffer,
        .uvc_buffer_size = UVC_MAX_FRAMESIZE_SIZE,
        .start_cb = camera_start_cb,
        .fb_get_cb = camera_fb_get_cb,
        .fb_return_cb = camera_fb_return_cb,
        .stop_cb = camera_stop_cb,
    };

    ESP_LOGI(TAG, "Format List");
    ESP_LOGI(TAG, "\tFormat(1) = %s", "MJPEG");
    ESP_LOGI(TAG, "Frame List");
    ESP_LOGI(TAG, "\tFrame(1) = %d * %d @%dfps", UVC_FRAMES_INFO[0][0].width, UVC_FRAMES_INFO[0][0].height, UVC_FRAMES_INFO[0][0].rate);
#if CONFIG_CAMERA_MULTI_FRAMESIZE
    ESP_LOGI(TAG, "\tFrame(2) = %d * %d @%dfps", UVC_FRAMES_INFO[0][1].width, UVC_FRAMES_INFO[0][1].height, UVC_FRAMES_INFO[0][1].rate);
    ESP_LOGI(TAG, "\tFrame(3) = %d * %d @%dfps", UVC_FRAMES_INFO[0][2].width, UVC_FRAMES_INFO[0][2].height, UVC_FRAMES_INFO[0][2].rate);
    ESP_LOGI(TAG, "\tFrame(3) = %d * %d @%dfps", UVC_FRAMES_INFO[0][3].width, UVC_FRAMES_INFO[0][3].height, UVC_FRAMES_INFO[0][3].rate);
#endif

    ESP_ERROR_CHECK(uvc_device_config(0, &config));
    ESP_ERROR_CHECK(uvc_device_init());

    while (1) {
#if CONFIG_CAMERA_MODULE_ESP_S3_EYE
#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 0, 0)
        static uint32_t close_time_ms = 0;
        static uint32_t open_time_ms = 0;
        EventBits_t bits = xEventGroupWaitBits(s_event_group, EYES_CLOSE_BIT | EYES_OPEN_BIT, pdTRUE, pdFALSE, pdMS_TO_TICKS(10));
        if (bits & EYES_CLOSE_BIT) {
            eyes_close(img);
            close_time_ms = 10;
            open_time_ms = 0;
            ESP_LOGI(TAG, "EYES CLOSE");
        } else if (bits & EYES_OPEN_BIT) {
            eyes_open(img);
            close_time_ms = 0;
            open_time_ms = 10;
            ESP_LOGI(TAG, "EYES OPEN");
        } else if (open_time_ms) {
            open_time_ms += 10;
            if (open_time_ms > 1000) {
                open_time_ms = 0;
                eyes_blink(img);
                ESP_LOGI(TAG, "EYES BLINK");
            }
        } else if (close_time_ms) {
            close_time_ms += 10;
            if (close_time_ms > 1000) {
                close_time_ms = 0;
                eyes_static(img);
                ESP_LOGI(TAG, "EYES STATIC");
            }
        }
#else
        vTaskDelay(pdMS_TO_TICKS(100));
#endif
#else
        vTaskDelay(pdMS_TO_TICKS(100));
#endif
    }
}

还可使用idf.py menuconfig按下面这样调整帧分辨率、帧速率和图像质量。
在这里插入图片描述
最后gc0308的成像：

测试代码

为了很方便的查看代码的输出，可以随便找个另外的项目，比如~/esp-iot-solution/examples/camera/test_framerate，将他的主程序文件take_picture.c修改为下方代码[to 自己：使用自己的开发板和乐鑫官方开发板时使用记得查看~/esp-iot-solution/examples/common_components/camera/include/camera_pin.h的摄像头引脚设置]：

/*
 * SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
 *
 * SPDX-License-Identifier: Apache-2.0
 */

#include <stdlib.h>
#include <string.h>
#include <inttypes.h>
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_timer.h"
#include "esp_log.h"
#include "camera_pin.h"
#include "esp_camera.h"
#include "usb_device_uvc.h"
#include "uvc_frame_config.h"


static const char *TAG = "usb_webcam";

#define CAMERA_XCLK_FREQ           20000000
#define CAMERA_FB_COUNT            2
#define CAMERA_MJPEG_SUPPORT false

#if CONFIG_IDF_TARGET_ESP32S3
#define UVC_MAX_FRAMESIZE_SIZE     (75*1024)
#else
#define UVC_MAX_FRAMESIZE_SIZE     (60*1024)
#endif

typedef struct {
    camera_fb_t *cam_fb_p;
    uvc_fb_t uvc_fb;
} fb_t;

static fb_t s_fb;

static esp_err_t camera_init(uint32_t xclk_freq_hz, pixformat_t pixel_format, framesize_t frame_size, int jpeg_quality, uint8_t fb_count)
{
    static bool inited = false;
    static uint32_t cur_xclk_freq_hz = 0;
    static pixformat_t cur_pixel_format = 0;
    static framesize_t cur_frame_size = 0;
    static int cur_jpeg_quality = 0;
    static uint8_t cur_fb_count = 0;

    if ((inited && cur_xclk_freq_hz == xclk_freq_hz && cur_pixel_format == pixel_format
            && cur_frame_size == frame_size && cur_fb_count == fb_count && cur_jpeg_quality == jpeg_quality)) {
        ESP_LOGD(TAG, "camera already inited");
        return ESP_OK;
    } else if (inited) {
        esp_camera_return_all();
        esp_camera_deinit();
        inited = false;
        ESP_LOGI(TAG, "camera RESTART");
    }

    camera_config_t camera_config = {
        .pin_pwdn = CAMERA_PIN_PWDN,
        .pin_reset = CAMERA_PIN_RESET,
        .pin_xclk = CAMERA_PIN_XCLK,
        .pin_sscb_sda = CAMERA_PIN_SIOD,
        .pin_sscb_scl = CAMERA_PIN_SIOC,

        .pin_d7 = CAMERA_PIN_D7,
        .pin_d6 = CAMERA_PIN_D6,
        .pin_d5 = CAMERA_PIN_D5,
        .pin_d4 = CAMERA_PIN_D4,
        .pin_d3 = CAMERA_PIN_D3,
        .pin_d2 = CAMERA_PIN_D2,
        .pin_d1 = CAMERA_PIN_D1,
        .pin_d0 = CAMERA_PIN_D0,
        .pin_vsync = CAMERA_PIN_VSYNC,
        .pin_href = CAMERA_PIN_HREF,
        .pin_pclk = CAMERA_PIN_PCLK,

        .xclk_freq_hz = xclk_freq_hz,
        .ledc_timer = LEDC_TIMER_0,
        .ledc_channel = LEDC_CHANNEL_0,

        .pixel_format = pixel_format,
        .frame_size = frame_size,

        .jpeg_quality = jpeg_quality,
        .fb_count = fb_count,
        .grab_mode = CAMERA_GRAB_WHEN_EMPTY,
        .fb_location = CAMERA_FB_IN_PSRAM
    };

    // initialize the camera sensor
    esp_err_t ret = esp_camera_init(&camera_config);
    if (ret != ESP_OK) {
        return ret;
    }

    // Get the sensor object, and then use some of its functions to adjust the parameters when taking a photo.
    // Note: Do not call functions that set resolution, set picture format and PLL clock,
    // If you need to reset the appeal parameters, please reinitialize the sensor.
    sensor_t *s = esp_camera_sensor_get();
    s->set_vflip(s, 1); // flip it back
    // initial sensors are flipped vertically and colors are a bit saturated
    if (s->id.PID == OV3660_PID) {
        s->set_brightness(s, 1); // up the blightness just a bit
        s->set_saturation(s, -2); // lower the saturation
    }

    if (s->id.PID == OV3660_PID || s->id.PID == OV2640_PID) {
        s->set_vflip(s, 1); // flip it back
    } else if (s->id.PID == GC0308_PID) {
        s->set_hmirror(s, 0);
    } else if (s->id.PID == GC032A_PID) {
        s->set_vflip(s, 1);
    }

    // Get the basic information of the sensor.
    camera_sensor_info_t *s_info = esp_camera_sensor_get_info(&(s->id));

    if (ESP_OK == ret && PIXFORMAT_JPEG == pixel_format && s_info->support_jpeg == true && CAMERA_MJPEG_SUPPORT == true) {
        ESP_LOGI(TAG, "JPEG format is supported");
        cur_xclk_freq_hz = xclk_freq_hz;
        cur_pixel_format = pixel_format;
        cur_frame_size = frame_size;
        cur_jpeg_quality = jpeg_quality;
        cur_fb_count = fb_count;
        inited = true;
    }  else if (ESP_OK == ret && PIXFORMAT_RGB565 == pixel_format && CAMERA_MJPEG_SUPPORT == false) {
        ESP_LOGI(TAG, "RGB565 format is supported");
        cur_xclk_freq_hz = xclk_freq_hz;
        cur_pixel_format = pixel_format;
        cur_frame_size = frame_size;
        cur_jpeg_quality = jpeg_quality;
        cur_fb_count = fb_count;
        inited = true;
    }
    else {
        ESP_LOGE(TAG, "Neither JPEG nor RGB565 formats are supported");
        return ESP_ERR_NOT_SUPPORTED;
    }

    return ret;
}

static void camera_stop_cb(void *cb_ctx)
{
    (void)cb_ctx;

    ESP_LOGI(TAG, "Camera Stop");
}

static esp_err_t camera_start_cb(uvc_format_t format, int width, int height, int rate, void *cb_ctx)
{
    (void)cb_ctx;
    ESP_LOGI(TAG, "Camera Start");
    ESP_LOGI(TAG, "Format: %d, width: %d, height: %d, rate: %d", format, width, height, rate);
    framesize_t frame_size = FRAMESIZE_QVGA;
    int jpeg_quality = 14;

    if (format != UVC_FORMAT_JPEG) {
        ESP_LOGE(TAG, "Only support MJPEG format");
        return ESP_ERR_NOT_SUPPORTED;
    }

    if (width == 320 && height == 240) {
        frame_size = FRAMESIZE_QVGA;
        jpeg_quality = 10;
    } else if (width == 480 && height == 320) {
        frame_size = FRAMESIZE_HVGA;
        jpeg_quality = 10;
    } else if (width == 640 && height == 480) {
        frame_size = FRAMESIZE_VGA;
        jpeg_quality = 12;
    } else if (width == 800 && height == 600) {
        frame_size = FRAMESIZE_SVGA;
        jpeg_quality = 14;
    } else if (width == 1280 && height == 720) {
        frame_size = FRAMESIZE_HD;
        jpeg_quality = 16;
    } else if (width == 1920 && height == 1080) {
        frame_size = FRAMESIZE_FHD;
        jpeg_quality = 16;
    } else {
        ESP_LOGE(TAG, "Unsupported frame size %dx%d", width, height);
        return ESP_ERR_NOT_SUPPORTED;
    }

    esp_err_t ret;
    if (CAMERA_MJPEG_SUPPORT)
    {
        ret = camera_init(CAMERA_XCLK_FREQ, PIXFORMAT_JPEG, frame_size, jpeg_quality, CAMERA_FB_COUNT);
    } else {
        ret = camera_init(CAMERA_XCLK_FREQ, PIXFORMAT_RGB565, frame_size, jpeg_quality, CAMERA_FB_COUNT);
    }

    if (ret != ESP_OK) {
        ESP_LOGE(TAG, "camera init failed");
        return ret;
    }


    return ESP_OK;
}

static uvc_fb_t* camera_fb_get_cb(void *cb_ctx)
{
    (void)cb_ctx;
    s_fb.cam_fb_p = esp_camera_fb_get();
    if (!s_fb.cam_fb_p) {
        return NULL;
    }
    if (s_fb.cam_fb_p->format == PIXFORMAT_JPEG) {
        printf("frame is jpeg format\n");
        s_fb.uvc_fb.buf = s_fb.cam_fb_p->buf;
        s_fb.uvc_fb.len = s_fb.cam_fb_p->len;
        s_fb.uvc_fb.format = s_fb.cam_fb_p->format;
    } else if (frame2jpg(s_fb.cam_fb_p, 60, &s_fb.uvc_fb.buf, &s_fb.uvc_fb.len)) {
        s_fb.uvc_fb.format = PIXFORMAT_JPEG;
        printf("JPEG compression success, JPEG: %d bytes\n", s_fb.uvc_fb.len);
    } else {
        s_fb.uvc_fb.buf = NULL;
        printf("JPEG compression failed\n");
        return NULL;
    }

    s_fb.uvc_fb.width = s_fb.cam_fb_p->width;
    s_fb.uvc_fb.height = s_fb.cam_fb_p->height;
    s_fb.uvc_fb.timestamp = s_fb.cam_fb_p->timestamp;

    if (s_fb.uvc_fb.len > UVC_MAX_FRAMESIZE_SIZE) {
        ESP_LOGE(TAG, "Frame size %d is larger than max frame size %d", s_fb.uvc_fb.len, UVC_MAX_FRAMESIZE_SIZE);
        esp_camera_fb_return(s_fb.cam_fb_p);
        return NULL;
    }
    return &s_fb.uvc_fb;
}

static void camera_fb_return_cb(uvc_fb_t *fb, void *cb_ctx)
{
    (void)cb_ctx;
    assert(fb == &s_fb.uvc_fb);
    esp_camera_fb_return(s_fb.cam_fb_p);
    #if !CAMERA_MJPEG_SUPPORT
        free(fb->buf);
    #endif
}

void app_main(void)
{
    ESP_LOGI(TAG, "Selected Camera Board %s", CAMERA_MODULE_NAME);
    uint8_t *uvc_buffer = (uint8_t *)malloc(UVC_MAX_FRAMESIZE_SIZE);
    if (uvc_buffer == NULL) {
        ESP_LOGE(TAG, "malloc frame buffer fail");
        return;
    }

    uvc_device_config_t config = {
        .uvc_buffer = uvc_buffer,
        .uvc_buffer_size = UVC_MAX_FRAMESIZE_SIZE,
        .start_cb = camera_start_cb,
        .fb_get_cb = camera_fb_get_cb,
        .fb_return_cb = camera_fb_return_cb,
        .stop_cb = camera_stop_cb,
    };

    ESP_LOGI(TAG, "Format List");
    ESP_LOGI(TAG, "\tFormat(1) = %s", "MJPEG");
    ESP_LOGI(TAG, "Frame List");
    ESP_LOGI(TAG, "\tFrame(1) = %d * %d @%dfps", UVC_FRAMES_INFO[0][1].width, UVC_FRAMES_INFO[0][1].height, UVC_FRAMES_INFO[0][1].rate);
    
    uvc_fb_t *fb = NULL;
    camera_start_cb(UVC_FORMAT_JPEG, UVC_FRAMES_INFO[0][1].width, UVC_FRAMES_INFO[0][1].height, UVC_FRAMES_INFO[0][1].rate, NULL);
    while (1) {
        fb = camera_fb_get_cb(NULL);
        camera_fb_return_cb(fb, NULL);
		vTaskDelay(pdMS_TO_TICKS(100));
    }
}

同目录下的idf_component.yml增加上usb_device_uvc的依赖：

dependencies:
  idf: ">=4.4.1"
  camera:
    version: "*"
    override_path: "../../../common_components/camera"
  esp32-camera:
    version: ">=2.0.3"
  usb_device_uvc:
    version: "1.1.*"
    override_path: "../../../../components/usb/usb_device_uvc"

烧录运行程序

idf.py set-target esp32s3
idf.py menuconfig # Camera Pin Configuration选择ESP-S3-EYE DevKit
idf.py flash monitor

输出如下
在这里插入图片描述

存在问题【仅记录，已解决更新了上面的代码】

1）直接用上面这样的代码在运行一小段时间后就会卡住，查看输出
在这里插入图片描述
参考ESP32出现喂狗失败处理办法，但我并没有更改超时看门狗的时间，只是在测试代码中增加了一句延时设置：

    while (1) {
        fb = camera_fb_get_cb(NULL);
        camera_fb_return_cb(fb, NULL);
		vTaskDelay(pdMS_TO_TICKS(100));
    }

2）但运行一段时间后又报了新的错误

E (26283) to_jpg: JPG buffer malloc failed
JPEG compression failed

assert failed: camera_fb_return_cb take_picture.c:242 (fb == &s_fb.uvc_fb)

这是因为frame2jpg中uint8_t * jpg_buf = (uint8_t *)_malloc(jpg_buf_len);申请了新内存，每帧结束需要释放掉该内存

static void camera_fb_return_cb(uvc_fb_t *fb, void *cb_ctx)
{
    (void)cb_ctx;
    assert(fb == &s_fb.uvc_fb);
    esp_camera_fb_return(s_fb.cam_fb_p);
    #if !CAMERA_MJPEG_SUPPORT
        free(fb->buf);
    #endif
}