DeepSORT C++版的一个bug

DeepSORT的官方python版实现是https://github.com/nwojke/deep_sort，C++版的DeepSORT中https://github.com/shaoshengsong/DeepSORT这个版本实现较早(这个版本又似乎是在GitHub - bitzy/DeepSort: c++ version of https://github.com/nwojke/deep_sort.这个版本上稍稍修改了下,yolov2改为yolov3)，github上可以看到不少人的DeepSORT都是在这个版本上做的集成和修改，绝大部分代码是照抄的，只是使用的目标检测模型或特征抽取模型不一样而已，或者对DeepSORT代码入口加了个封装类DeepSort，将特征抽取模型部分弄成支持TensorRT的onnx解析方式或者直接读取解析onnx后生成的engine文件(例如:GitHub - GesilaA/deepsort_tensorrt: This is a TensorRT based deepsort project  GitHub - mellivoraPKU/yolov5_trt_deepsort   https://github.com/RichardoMrMu/yolov5-deepsort-tensorrt)，因此把这个版本里的Bug也就都带过去了，凡是基于bitzy/DeepSort或者shaoshengsong/DeepSORT这个序列的版本都存在这个问题。

跑这个C++版本以及其他基于这版的DeepSORT时，长时间跑肯定会因为Eigen Block的非法索引访问而发生崩溃，只是这个错误不会经常出现，具体的错误类似如下:

#2  0x0000007f92976b84 in __assert_fail_base (fmt=0x7f92a71c10 "%s%s%s:%u: %s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x7f306fc110 "startRow >= 0 && blockRows >= 0 && startRow <= xpr.rows() - blockRows && startCol >= 0 && blockCols >= 0 && startCol <= xpr.cols() - blockCols", file=file@entry=0x7f306fbfb8 "/usr/include/eigen3/Eigen/src/Core/Block.h", line=line@entry=147, function=function@entry=0x7f306fd908 <Eigen::Block<Eigen::Matrix<float, -1, 2048, 1, -1, 2048>, -1, -1, false>::Block(Eigen::Matrix<float, -1, 2048, 1, -1, 2048>&, long, long, long, long)::__PRETTY_FUNCTION__> "Eigen::Block<XprType, BlockRows, BlockCols, InnerPanel>::Block(XprType&, Eigen::Index, Eigen::Index, Eigen::Index, Eigen::Index) [with XprType = Eigen::Matrix<float, -1, 2048, 1>; int BlockRows = -1; "...) at assert.c:92

#3  0x0000007f92976c04 in __GI___assert_fail (assertion=0x7f306fc110 "startRow >= 0 && blockRows >= 0 && startRow <= xpr.rows() - blockRows && startCol >= 0 && blockCols >= 0 && startCol <= xpr.cols() - blockCols", file=0x7f306fbfb8 "/usr/include/eigen3/Eigen/src/Core/Block.h", line=147, function=0x7f306fd908 <Eigen::Block<Eigen::Matrix<float, -1, 2048, 1, -1, 2048>, -1, -1, false>::Block(Eigen::Matrix<float, -1, 2048, 1, -1, 2048>&, long, long, long, long)::__PRETTY_FUNCTION__> "Eigen::Block<XprType, BlockRows, BlockCols, InnerPanel>::Block(XprType&, Eigen::Index, Eigen::Index, Eigen::Index, Eigen::Index) [with XprType = Eigen::Matrix<float, -1, 2048, 1>; int BlockRows = -1; "...) at assert.c:101

#4  0x0000007f306059c4 in Eigen::Block<Eigen::Matrix<float, -1, 2048, 1, -1, 2048>, -1, -1, false>::Block(Eigen::Matrix<float, -1, 2048, 1, -1, 2048>&, long, long, long, long) (this=0x7f06d50580, xpr=..., startRow=2, startCol=0, blockRows=97, blockCols=2048) at /usr/include/eigen3/Eigen/src/Core/Block.h:146
#5  0x0000007f30604978 in Eigen::DenseBase<Eigen::Matrix<float, -1, 2048, 1, -1, 2048> >::block(long, long, long, long) (this=0x7ef4041198, startRow=2, startCol=0, blockRows=97, blockCols=2048) at /usr/include/eigen3/Eigen/src/plugins/BlockMethods.h:66

#6  0x0000007f3063acf0 in NearNeighborDisMetric::partial_fit(std::vector<std::pair<int, Eigen::Matrix<float, -1, 2048, 1, -1, 2048> >, std::allocator<std::pair<int, Eigen::Matrix<float, -1, 2048, 1, -1, 2048> > > >&, std::vector<int, std::allocator<int> >&) (this=0x55aded9310, tid_feats=std::vector of length 5, capacity 8 = {...}, active_targets=std::vector of length 5, capacity 8 = {...}) at nn_matching.cpp:68

#7  0x0000007f30627d44 in tracker::update(std::pair<std::vector<CLSCONF, std::allocator<CLSCONF> >, std::vector<DETECTION_ROW, std::allocator<DETECTION_ROW> > > const&) (this=0x55aeed85f0, detectionsv2={...}) at tracker.cpp:106
#8  0x0000007f306a75e8 in DeepSort::sort(cv::Mat&, std::pair<std::vector<CLSCONF, std::allocator<CLSCONF> >, std::vector<DETECTION_ROW, std::allocator<DETECTION_ROW> > >&) (this=0x55aeed8550, frame=..., detectionsv2={...}) at deepsort.cpp:104

#9  0x0000007f306a721c in DeepSort::sort(cv::Mat&, std::vector<DetectBox, std::allocator<DetectBox> >&) (this=0x55aeed8550, frame=..., dets=std::vector of length 2, capacity 2 = {...})
    at deepsort.cpp:51

因为不同的版本的代码可能稍微做了修改，发生的行数可能有所不同，但是nn_matching.cpp里这些代码没有修改的话，一定会出现类似上面的崩溃:

if(samples.find(track_id) != samples.end()) {//append
          int oldSize = samples[track_id].rows();
          int addSize = newFeatOne.rows();
          int newSize = oldSize + addSize;

          if(newSize <= this->budget) {
              FEATURESS newSampleFeatures(newSize, 128);
              newSampleFeatures.block(0,0, oldSize, 128) = samples[track_id];
              newSampleFeatures.block(oldSize, 0, addSize, 128) = newFeatOne;
              samples[track_id] = newSampleFeatures;
            } else {

                if(oldSize < this->budget) {//original space is not enough;
                  FEATURESS newSampleFeatures(this->budget, 128);
                  if(addSize >= this->budget) {
                      newSampleFeatures = newFeatOne.block(0, 0, this->budget, 128);
                    } else {
                      newSampleFeatures.block(0, 0, this->budget-addSize, 128) =
                          samples[track_id].block(addSize-1, 0, this->budget-addSize, 128).eval();
                      newSampleFeatures.block(this->budget-addSize, 0, addSize, 128) = newFeatOne;
                    }
                  samples[track_id] = newSampleFeatures;
                } else {//original space is ok;
                  if(addSize >= this->budget) {
                      samples[track_id] = newFeatOne.block(0,0, this->budget, 128);
                    } else {
                      samples[track_id].block(0, 0, this->budget-addSize, 128) =
                          samples[track_id].block(addSize-1, 0, this->budget-addSize, 128).eval();
                      samples[track_id].block(this->budget-addSize, 0, addSize, 128) = newFeatOne;
                    }
                }
    }

原因就是上面代码中有两处代码是这样写的:

newSampleFeatures.block(0, 0, this->budget-addSize, 128) =
    samples[track_id].block(addSize-1, 0, this->budget-addSize, 128).eval();

...

samples[track_id].block(0, 0, this->budget-addSize, 128) =
   samples[track_id].block(addSize-1, 0, this->budget-addSize, 128).eval();

samples[track_id].block(this->budget-addSize, 0, addSize, 128) = newFeatOne;

这里写错了！上面的代码的本意是把newSampleFeatures的后addSize用来存储需要加进来的新的feature数据newFeatOne，前面剩下的this->budget-addSize行用来存samples中的后this->budget-addSize行特征数据，那samples中取值开始的行数应该是oldsize - (this->budget-addSize)而不是作者写的addSize-1 !!!，所以上面的代码应该修改为下面这样才正确：

newSampleFeatures.block(0, 0, this->budget-addSize, 128) =
    samples[track_id].block(oldSize -(this->budget-addSize), 0, this->budget - addSize, 128).eval();

samples[track_id].block(0, 0, oldSize - addSize, 128) =
   samples[track_id].block(addSize, 0, oldSize - addSize, 128).eval();

samples[track_id].block(oldSize - addSize, 0, addSize, 128) = newFeatOne;

这部分完整代码修改后应该是这样:

if(samples.find(track_id) != samples.end()) {//append
          int oldSize = samples[track_id].rows();
          int addSize = newFeatOne.rows();
          int newSize = oldSize + addSize;

          if(newSize <= this->budget) {
              FEATURESS newSampleFeatures(newSize, 128);
              newSampleFeatures.block(0,0, oldSize, 128) = samples[track_id];
              newSampleFeatures.block(oldSize, 0, addSize, 128) = newFeatOne;
              samples[track_id] = newSampleFeatures;
            } else {

                if(oldSize < this->budget) {//original space is not enough;
                  FEATURESS newSampleFeatures(this->budget, 128);
                  if(addSize >= this->budget) {
                      newSampleFeatures = newFeatOne.block(0, 0, this->budget, 128);
                    } else {
                      newSampleFeatures.block(0, 0, this->budget-addSize, 128) =
                          samples[track_id].block(oldSize -(this->budget-addSize), 0, this->budget-addSize, 128).eval();
                      newSampleFeatures.block(this->budget-addSize, 0, addSize, 128) = newFeatOne;
                    }
                  samples[track_id] = newSampleFeatures;
                } else {//original space is ok;
                  if(addSize >= this->budget) {
                      samples[track_id] = newFeatOne.block(0,0, this->budget, 128);
                    } else {
                      samples[track_id].block(0, 0, oldSize-addSize, 128) =
                          samples[track_id].block(addSize, 0, oldSize-addSize, 128).eval();
                      samples[track_id].block(oldSize - addSize, 0, addSize, 128) = newFeatOne;
                    }
                }
    }