在安卓上运行yolov8目标检测模型（ncnn）

在安卓上运行yolov8目标检测模型（ncnn）

关于

• 首次发表日期：2024-07-25
• 本人不会Java和Android，如有错误，敬请指出
• 主要是整理一下以下资料
  • https://medium.com/@gary.tsai.advantest/top-tutorials-for-deploying-custom-yolov8-on-android-%EF%B8%8F-dd6746afc1e6
  • https://github.com/FeiGeChuanShu/ncnn-android-yolov8
  • https://github.com/Digital2Slave/ncnn-android-yolov8-seg/wiki/Convert-yolov8%E2%80%90seg-to-ncnn-model-step-by-step

准备环境

首先准备conda环境，由于使用的是较旧版本的yolov8库，这儿也采用相应版本的pytorch

conda create -n ncnn_yolov8 python=3.8
conda activate ncnn_yolov8

pip install torch==1.8.2 torchvision==0.9.2 torchaudio==0.8.2 --extra-index-url https://download.pytorch.org/whl/lts/1.8/cu111

git clone https://github.com/ultralytics/ultralytics
cd ultralytics
git checkout b9b0fd8bf409c822b7fcb21d65722b242f5307fc

pip install -r  requirements.txt
pip install -e .

训练

第一步，先更新yolov8的设置，主要是配置一下数据集文件夹：

添加update_settings.py文件：

# update_settings.py
from ultralytics import settings
import os

current_folder = os.path.dirname(__file__)
datasets_dir = '/mnt/d/0-Datasets/ultralytics_datasets_dir/datasets'
weights_dir = os.path.join(current_folder, "weights")
runs_dir = os.path.join(current_folder, "runs")

settings.update({'datasets_dir': datasets_dir})
settings.update({'weights_dir': weights_dir})
settings.update({'runs_dir': runs_dir})

os.makedirs(weights_dir, exist_ok=True)
os.makedirs(runs_dir, exist_ok=True)

python update_settings.py

然后进行训练，这儿以coco8数据集为例。

添加train.py文件：

# train.py
from ultralytics import YOLO
import torch


resume = False
if not resume:
    # Create a new YOLO model from scratch
    model = YOLO('yolov8n.yaml').load('yolov8n.pt')
else:
    model = YOLO('runs/detect/train/weights/last.pt')

device_count = torch.cuda.device_count()
model.train(data='coco8.yaml', imgsz=640, epochs=10, resume=resume, batch = 32 * device_count, device=list(range(0,device_count)))

python train.py

导出权重文件

导出onnx文件

from ultralytics import YOLO

model = YOLO("runs/detect/train4/weights/best.pt")

# Export model
success = model.export(task="detection", format="onnx", opset=12, imgsz=640, simplify=True)

安装ncnn

首先需要确保cuda的版本是11.8。

我同时安装了12.2和11.8，默认版本配置在.bashrc文件中，之前默认的版本是12.2，所以我修改了一下：

export PATH=/usr/local/cuda-11.8/bin${PATH:+:${PATH}}

source ~/.bashrc

然后安装：

cd ..

git clone https://github.com/Tencent/ncnn.git
cd ncnn
git submodule update --init
sudo apt install build-essential git cmake libprotobuf-dev protobuf-compiler libvulkan-dev vulkan-utils libopencv-dev

# build part
mkdir build && cd build
cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ ..
make -j$(nproc)
sudo make install

导出ncnn

修改 ultralytics/ultralytics/nn/modules/block.py 中 C2f 的 forward 方法为：

    def forward(self, x):
        # """Forward pass through C2f layer."""
        # y = list(self.cv1(x).chunk(2, 1))
        # y.extend(m(y[-1]) for m in self.m)
        # return self.cv2(torch.cat(y, 1))
        # !< https://github.com/FeiGeChuanShu/ncnn-android-yolov8
        x = self.cv1(x)
        x = [x, x[:, self.c:, ...]]
        x.extend(m(x[-1]) for m in self.m)
        x.pop(1)
        return self.cv2(torch.cat(x, 1))

修改 ultralytics/ultralytics/nn/modules/head.py 中 Detect 的 forward 方法为：

    def forward(self, x):
            shape = x[0].shape  # BCHW
            for i in range(self.nl):
                x[i] = torch.cat((self.cv2[i](x[i]), self.cv3[i](x[i])), 1)
            if self.training:
                return x
            elif self.dynamic or self.shape != shape:
                self.anchors, self.strides = (x.transpose(0, 1) for x in make_anchors(x, self.stride, 0.5))
                self.shape = shape
            
            # box, cls = torch.cat([xi.view(shape[0], self.no, -1) for xi in x], 2).split((self.reg_max * 4, self.nc), 1)
            # dbox = dist2bbox(self.dfl(box), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides
            # y = torch.cat((dbox, cls.sigmoid()), 1)
            # return y if self.export else (y, x)

            return torch.cat([xi.view(shape[0], self.no, -1) for xi in x], 2).permute(0, 2, 1)

导出

cd ultralytics/runs/detect/train/weights
onnx2ncnn best.onnx best.param best.bin

这一步将生成best.param和best.bin这2个文件

配置android demo

这儿将使用 https://github.com/FeiGeChuanShu/ncnn-android-yolov8 提供的android demo

git clone https://github.com/FeiGeChuanShu/ncnn-android-yolov8
cd ncnn-android-yolov8/ncnn-android-yolov8

下载ncnn-YYYYMMDD-android-vulkan.zip，移到到 app/src/main/jni/ 中，并解压

下载opencv-mobile，移到到 app/src/main/jni/ 中，并解压

打开 app/src/main/jni/CMakeLists.txt，确保 opencv 和 ncnn 的版本与下载的版本匹配，修改后类似如下：

project(yolov8ncnn)

cmake_minimum_required(VERSION 3.10)

set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/opencv-mobile-4.6.0-android/sdk/native/jni)
find_package(OpenCV REQUIRED core imgproc)

set(ncnn_DIR ${CMAKE_SOURCE_DIR}/ncnn-20220420-android-vulkan/${ANDROID_ABI}/lib/cmake/ncnn)
find_package(ncnn REQUIRED)

add_library(yolov8ncnn SHARED yolov8ncnn.cpp yolo.cpp ndkcamera.cpp)

target_link_libraries(yolov8ncnn ncnn ${OpenCV_LIBS} camera2ndk mediandk)

将 assets 文件夹中的权重文件备份，然后移入之前导出的ncnn权重，并重命名为 yolov8n.bin和 yolov8n.param

根据 yolov8n.param 文件的最后一行：

Permute          Transpose_227            1 1 380 output0 0=1

修改 app\src\main\jni\yolo.cpp 中的输出层名称，修改后的那一行如下：

ex.extract("output0", out);

然后修改 app\src\main\jni\yolo.cpp 中的 generate_proposals函数中定义的num_class和Yolo::draw中定义的class_names，使其和自己的模型一致

最后使用android studio打开，配置好ndk依赖等，运行