tensorrt yolov5 批量预测学习笔记

目录

编译引擎：

完整调试博客：

---

多张图片预测：

https://github.com/noahmr/yolov5-tensorrt

 https://github.com/noahmr/yolov5-tensorrt/blob/main/src/yolov5_detector.cpp

Result Detector::detectBatch(const std::vector<cv::cuda::GpuMat>& images, 
                    std::vector<std::vector<Detection>>* out,
                    int flags) noexcept

本地项目名称：

yolov5-tensorrt_noahmr

批量预测代码：

Result Detector::detectBatch(const std::vector<cv::Mat>& images, 
                    std::vector<std::vector<Detection>>* out,
                    int flags) noexcept
{
    if(!isEngineLoaded()){
        if(_logger){
            _logger->log(LOGGING_ERROR, "[Detector] detectBatch() failure: no "
                        "engine loaded");
        }
        return RESULT_FAILURE_NOT_LOADED;
    }
    if(images.size() == 0){
        _logger->log(LOGGING_ERROR, "[Detector] detectBatch() failure: list "
                        "of inputs is empty");
        return RESULT_FAILURE_INVALID_INPUT;
    }
    if((int)images.size() > _batchSize()){
        _logger->logf(LOGGING_ERROR, "[Detector] detectBatch() failure: "
                        "specified %d images, but batch size is %i",
                        (unsigned int)images.size(), _batchSize());
        return RESULT_FAILURE_INVALID_INPUT;
    }
    const int numProcessed = MIN((int)images.size(), _batchSize());

    /**     Pre-processing      **/
    if(!_preprocessor->setup(_inputBinding.dims(), flags, _batchSize(), 
                            (float*)_deviceMemory.at(_inputBinding.index()) ))
    {
        _logger->log(LOGGING_ERROR, "[Detector] detectBatch() failure: could "
                        "not set up pre-processor");
        return RESULT_FAILURE_OTHER;
    }

    for(int i = 0; i < numProcessed; ++i){
        if(!_preprocessor->process(i, images[i], i == numProcessed - 1))
        {
            _logger->logf(LOGGING_ERROR, "[Detector] detectBatch() "
                            "failure: preprocessing for image %i failed", i);
            return RESULT_FAILURE_OTHER;
        }
    }

    return _detectBatch(numProcessed, out);

预处理代码：

bool Preprocessor::process(const int& index, 
            const cv::cuda::GpuMat& input, const bool& last) noexcept
{
    YOLOV5_UNUSED(input);
    YOLOV5_UNUSED(last);

    if(_transforms.size() < (unsigned int)index+1)
    {
        try
        {
            _transforms.resize(index + 1);
        }
        catch(const std::exception& e)
        {
            _logger->logf(LOGGING_ERROR, "[Preprocessor] process() "
                            "failure: got exception setting up transforms: "
                            "%s", e.what());
            return false;
        }
    }
    return true;
}

setup out:

Result DeviceMemory::setup(const std::shared_ptr<Logger>& logger, 
                    std::unique_ptr<nvinfer1::ICudaEngine>& engine,
                    DeviceMemory* output) noexcept
{
    const int32_t nbBindings = engine->getNbBindings();
    for(int i = 0; i < nbBindings; ++i)
    {
        const nvinfer1::Dims dims = engine->getBindingDimensions(i); 
        const int volume = dimsVolume(dims);

        try
        {
            output->_memory.push_back(nullptr);
        }
        catch(const std::exception& e)
        {
            logger->logf(LOGGING_ERROR, "[DeviceMemory] setup() failure: "
                        "exception: %s", e.what()); 
            return RESULT_FAILURE_ALLOC;
        }
        void** ptr = &output->_memory.back();

        auto r = cudaMalloc(ptr, volume * sizeof(float));
        if(r != 0 || *ptr == nullptr)
        {
            logger->logf(LOGGING_ERROR, "[DeviceMemory] setup() failure: "
                        "could not allocate device memory: %s", 
                        cudaGetErrorString(r));
            return RESULT_FAILURE_CUDA_ERROR;
        }

    }
    return RESULT_SUCCESS;
}

编译引擎：

设置BatchSize：

m_BatchSize = std::stoi(trim(block.at("batch")));

	m_Builder->setMaxBatchSize(m_BatchSize);
	config->setMaxWorkspaceSize(1<<20);

本地项目：yolov5-5.0_voc

很感谢这个作者：

GitHub - enazoe/yolo-tensorrt: TensorRT8.Support Yolov5n,s,m,l,x .darknet -> tensorrt. Yolov4 Yolov3 use raw darknet *.weights and *.cfg fils. If the wrapper is useful to you,please Star it.

https://github.com/enazoe/yolo-tensorrt/blob/c4d72605f83d547081cc30c3b71458001826191d/modules/class_yolo_detector.hpp

cv::Mat trtInput = blobFromDsImages(vec_ds_images, _p_net->getInputH(),_p_net->getInputW());

yolov5-v6:

• inequal net width and height
•  batch inference
•  support FP32,FP16,INT8

下载后：yolo-tensorrt-TRT8.zip

入口：

https://github.com/enazoe/yolo-tensorrt/blob/c4d72605f83d547081cc30c3b71458001826191d/samples/sample_detector.cpp

完整调试博客：

yolov5转tensorrt c++_jacke121的专栏-CSDN博客

https://github.com/OpenJetson/tensorrt-yolov5/blob/main/yolov5.cpp

for循环添加数据

static float data[BATCH_SIZE * 3 * INPUT_H * INPUT_W];

cudaMemcpy 效率不高：

https://github.com/Wulingtian/yolov5_tensorrt_int8/blob/master/yolov5s_infer.cc

https://github.com/jacke121/yolo-tensorrt

	void detect(const std::vector<cv::Mat>	&vec_image,
				std::vector<BatchResult> &vec_batch_result)
	{
		Timer timer;
		std::vector<DsImage> vec_ds_images;
		vec_batch_result.clear();
		vec_batch_result.resize(vec_image.size());
		for (const auto &img:vec_image)
		{
			vec_ds_images.emplace_back(img, _vec_net_type[_config.net_type], _p_net->getInputH(), _p_net->getInputW());
		}
		cv::Mat trtInput = blobFromDsImages(vec_ds_images, _p_net->getInputH(),_p_net->getInputW());
		timer.out("pre");
		_p_net->doInference(trtInput.data, vec_ds_images.size());
		timer.reset();
		for (uint32_t i = 0; i < vec_ds_images.size(); ++i)
		{
			auto curImage = vec_ds_images.at(i);
			auto binfo = _p_net->decodeDetections(i, curImage.getImageHeight(), curImage.getImageWidth());
			auto remaining = nmsAllClasses(_p_net->getNMSThresh(),
				binfo,
				_p_net->getNumClasses(),
				_vec_net_type[_config.net_type]);
			if (remaining.empty())
			{
				continue;
			}
			std::vector<Result> vec_result(0);
			for (const auto &b : remaining)
			{
				Result res;
				res.id = b.label;
				res.prob = b.prob;
				const int x = b.box.x1;
				const int y = b.box.y1;
				const int w = b.box.x2 - b.box.x1;
				const int h = b.box.y2 - b.box.y1;
				res.rect = cv::Rect(x, y, w, h);
				vec_result.push_back(res);
			}
			vec_batch_result[i] = vec_result;
		}
		timer.out("post");