DNN模型压缩：量化实验之ncnn

ncnn

ncnn 是一个为手机端极致优化的高性能神经网络前向计算框架。ncnn 从设计之初深刻考虑手机端的部署和使用。无第三方依赖，跨平台，手机端 cpu 的速度快于目前所有已知的开源框架。基于 ncnn，开发者能够将深度学习算法轻松移植到手机端高效执行，开发出人工智能 APP，将 AI 带到你的指尖。ncnn 目前已在腾讯多款应用中使用，如 QQ，Qzone，微信，天天P图等。

1. 安装

• 参考：安装 与 使用

• 下载ncnn
  git clone https://github.com/Tencent/ncnn

• 进入ncnn根目录cd<ncnn-root-dir>

• 注：在CMakeLists.txt中取消注释add_subdirectory(examples)，以便编译examples中的cpp文件。

• 执行以下命令，编译ncnn:

$ mkdir -p build
$ cd build
$ cmake ..
$ make -j4
$ make install

这样就得到了build/examples文件下的多个模型的可执行文件。

2. 使用

Squeezenet进行分类测试

• 移动ncnn所需的param和bin文件到build/examples/

$ cp examples/squeezenet_v1.1.param build/examples/
$ cp examples/squeezenet_v1.1.bin build/examples/

• 分类预测

$ cd build/examples/
$ ./squeezenet dog.jpg 

• 预测结果

258 = 0.191417
257 = 0.109412
151 = 0.060365

3. ncnn量化：8位整型的实现

• Initial the int8 quantize inference implement
• 工具：caffe-int8-convert-tools
• 8-bit Inference with TensorRT

量化cifar_small模型

cifar_small模型是DarkNet框架的小型网络，它包含7层卷积。其中，将其模型文件cifar_small.cfg改写成caffe框架的配置文件请参考这里。

• 准备caffe网络和模型

train.prototxt
deploy.prototxt
snapshot_10000.caffemodel

• 执行量化脚本

python caffe-int8-convert-tool-dev.py --proto=cifar_small-master/cifar_small_deploy.prototxt --model=cifar_small-master/cifar_small_iter_10000.caffemodel --mean 125.3 123.0 113.9 --norm=1 --images=cifar_test_100/ --output=cifar_small.table --group=1  --gpu=0 

• 得到量化结果：

cifar_test_100//128_dog.png forward time : 0.001 s
loop stage 2 : 54
add cost 0.003 s
normalize_distribution 168664 2048
caffe-int8-convert-tool-dev.py:193: RuntimeWarning: divide by zero encountered in true_divide
  return np.sum(dist_a[nonzero_inds] * np.log(dist_a[nonzero_inds] / dist_b[nonzero_inds]))
conv1                group : 0     bin : 2034     threshold : 140.169904 interval : 0.068896   scale : 0.906043  
normalize_distribution 450560 2048
conv2                group : 0     bin : 1189     threshold : 3.555792   interval : 0.002989   scale : 35.716380 
normalize_distribution 225280 2048
conv3                group : 0     bin : 1545     threshold : 2.780417   interval : 0.001799   scale : 45.676601 
normalize_distribution 225280 2048
conv4                group : 0     bin : 1569     threshold : 1.943042   interval : 0.001238   scale : 65.361413 
normalize_distribution 112640 2048
conv5                group : 0     bin : 1588     threshold : 2.604649   interval : 0.001640   scale : 48.758978 
normalize_distribution 450560 2048
conv6                group : 0     bin : 1287     threshold : 1.208493   interval : 0.000939   scale : 105.089525
normalize_distribution 225280 2048
conv7                group : 0     bin : 1175     threshold : 5.926937   interval : 0.005042   scale : 21.427592 

Caffe Int8 Calibration table create success, it's cost 0:00:41.287920, best wish for your INT8 inference has a low accuracy loss...\(^▽^)/...2333...

生成 cifarsmall.table文件

• 统计量化结果

参考squeezent.cpp并利用darknet的validate_classifier_single函数进行批量测试：

// Tencent is pleased to support the open source community by making ncnn available.                                                                                                                    
  2 //
  3 // Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  4 //
  5 // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
  6 // in compliance with the License. You may obtain a copy of the License at
  7 //
  8 // https://opensource.org/licenses/BSD-3-Clause
  9 //
 10 // Unless required by applicable law or agreed to in writing, software distributed
 11 // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 12 // CONDITIONS OF ANY KIND, either express or implied. See the License for the
 13 // specific language governing permissions and limitations under the License.
 14 
 15 #include <stdio.h>
 16 #include <algorithm>
 17 #include <vector>
 18 #include <opencv2/core/core.hpp>
 19 #include <opencv2/highgui/highgui.hpp>
 20 
 21 #include "darknet.h"
 22 #include "net.h"
 23 
 24 static int detect_cifarsmall(const cv::Mat& bgr, std::vector<float>& cls_scores, char *param, char *bin)
 25 {
 26     ncnn::Net cifarsmall;
 27     // cifarsmall.load_param("cifar_small.param");
 28     // cifarsmall.load_model("cifar_small.bin");
 29     cifarsmall.load_param(param);
 30     cifarsmall.load_model(bin);
 31     
 32     ncnn::Mat in = ncnn::Mat::from_pixels_resize(bgr.data, ncnn::Mat::PIXEL_BGR, bgr.cols, bgr.rows, 28, 28);
 33     
 34     const float mean_vals[3] = {104.f, 117.f, 123.f};
 35     in.substract_mean_normalize(mean_vals, 0);
 36     
 37     ncnn::Extractor ex = cifarsmall.create_extractor();
 38     
 39     ex.input("data", in);
 40     
 41     ncnn::Mat out;
 42     ex.extract("prob", out);
 43     
 44     cls_scores.resize(out.w);
 45     for (int j=0; j<out.w; j++)
 46     {   
 47         cls_scores[j] = out[j];
 48     }
 49     
 50     return 0;
 51 }
 52 
 53 static int print_topk(const std::vector<float>& cls_scores, int topk)
 54 {
 55     // partial sort topk with index
 56     int size = cls_scores.size();
 57     std::vector< std::pair<float, int> > vec;
 58     vec.resize(size);
 59     for (int i=0; i<size; i++)
 60     {
 61         vec[i] = std::make_pair(cls_scores[i], i);
 62     }
 63 
 64     std::partial_sort(vec.begin(), vec.begin() + topk, vec.end(),
 65                       std::greater< std::pair<float, int> >());
 66 
 67     // print topk and score
 68     for (int i=0; i<topk; i++)
 69     {
 70         float score = vec[i].first;
 71         int index = vec[i].second;                                                                                                                                                                      
 72         fprintf(stderr, "%d = %f\n", index, score);
 73     }
 74 
 75     return 0;
 76 }
 77 
 78 void validate_classifier_single(char *datacfg, char *param, char *bin, char *labelfile, char *validfile)
 79 {
 80     int i, j;
 81     srand(time(0));
 82 
 83     // 其实不需要，因为下面会用到options。
 84     list *options = read_data_cfg(datacfg);
 85 
 86     char *label_list = option_find_str(options, "labels", labelfile);
 87     char *valid_list = option_find_str(options, "valid", validfile);
 88     // char *label_list = option_find_str(options, "labels", "data/labels.list");
 89     // char *valid_list = option_find_str(options, "valid", "data/train.list");
 90     int classes = option_find_int(options, "classes", 2);
 91     int topk = option_find_int(options, "top", 1);
 92 
 93     char **labels = get_labels(label_list);
 94     list *plist = get_paths(valid_list);
 95 
 96     char **paths = (char **)list_to_array(plist);
 97     int m = plist->size;
 98     free_list(plist);
 99 
100     float avg_acc = 0;
101     float avg_topk = 0;
102     int *indexes = (int*)calloc(topk, sizeof(int));
103     // 统计每类的分类正确率
104     float *mycls = (float *)calloc(classes, sizeof(float));
105     // 初始化二维数组，统计包含1000个样本的测试集的准确率
106     int cls[2][1000];
107     for (i = 0; i < 1000; i++) {
108         cls[0][i] = 0;
109         cls[1][i] = 0;
110     }
111     for(i = 0; i < m; ++i){
112         int class1 = -1;
113         char *path = paths[i];
114         for(j = 0; j < classes; ++j){
115             if(strstr(path, labels[j])){
116                 class1 = j;
117                 break;
118             }
119         }
120 
121         // ncnn的预测
122         const char* imagepath = paths[i];
123 
124         cv::Mat m = cv::imread(imagepath, CV_LOAD_IMAGE_COLOR);
125         if (m.empty())
126         {
127             fprintf(stderr, "cv::imread %s failed\n", imagepath);
128             continue;
129         }
130         std::vector<float> cls_scores;
131         detect_cifarsmall(m, cls_scores, param, bin);
132 
133         print_topk(cls_scores, 3);
134                                                                                                                                                                                                         
135         for(j=0; j < classes; ++j){
136           mycls[j] = cls_scores[j];
137         }
138         top_k(mycls, classes, topk, indexes);
139         // 统计每一类分类正确的次数
140         if(indexes[0] == class1) {
141             avg_acc += 1;
142             cls[1][class1]++;
143             cls[0][class1]++;
144         }else{
145             cls[0][class1]++;
146         }
147 
148         for(j = 0; j < topk; ++j){
149             if(indexes[j] == class1) avg_topk += 1;
150         }
151
152         printf("%s, %d, %f, %f, \n", paths[i], class1, mycls[0], mycls[1]);
153         printf("%d: top 1: %f, top %d: %f\n", i, avg_acc/(i+1), topk, avg_topk/(i+1));
154     }
155     for (i = 0; i < classes; i++) {
156         printf("[%d] %d %d %f \n", i, cls[0][i]);
157     }
158 }
159 
160 int main(int argc, char **argv)
161 {
162   if (argc != 6)
163   {
164     fprintf(
165         stderr,
166         "Usage: %s [datacfg] [ncnn.param] [ncnn.bin] [label.list] [train.list]\n",
167         argv[0]);
168     return -1;
169   }
170 
171 
172   validate_classifier_single(argv[1], argv[2], argv[3], argv[4], argv[5]);
173 
174   return 0;
175 }

修改example/CMakeLists.txt如下，假设你的tools文件夹已经生成了darknet相关文件，添加头文件darknet.h、libdarknet.so以及cifarsmall 链接库。

  1                                                                                                                                                                                                         
  2 find_package(OpenCV QUIET COMPONENTS core highgui imgproc imgcodecs)
  3 if(NOT OpenCV_FOUND)
  4     find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc)
  5 endif()
  6 
  7 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../src)
  8 include_directories(${CMAKE_CURRENT_BINARY_DIR}/../src)
  9 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../tools/darknet/darknet2ncnn/darknet/include)
 10 
 11 add_executable(squeezenet squeezenet.cpp)
 12 target_link_libraries(squeezenet ncnn ${OpenCV_LIBS})
 13 
 14 add_executable(fasterrcnn fasterrcnn.cpp)
 15 target_link_libraries(fasterrcnn ncnn ${OpenCV_LIBS})
 16 
 17 add_executable(rfcn rfcn.cpp)
 18 target_link_libraries(rfcn ncnn ${OpenCV_LIBS})
 19 
 20 add_executable(yolov2 yolov2.cpp)
 21 target_link_libraries(yolov2 ncnn ${OpenCV_LIBS})
 22 
 23 add_executable(yolov3 yolov3.cpp)
 24 target_link_libraries(yolov3 ncnn ${OpenCV_LIBS})
 25 
 26 add_executable(mobilenetv2ssdlite mobilenetv2ssdlite.cpp)
 27 target_link_libraries(mobilenetv2ssdlite ncnn ${OpenCV_LIBS})
 28 
 29 add_executable(mobilenetssd mobilenetssd.cpp)
 30 target_link_libraries(mobilenetssd ncnn ${OpenCV_LIBS})
 31 
 32 add_executable(squeezenetssd squeezenetssd.cpp)
 33 target_link_libraries(squeezenetssd ncnn ${OpenCV_LIBS})
 34 
 35 add_executable(shufflenetv2 shufflenetv2.cpp)
 36 target_link_libraries(shufflenetv2 ncnn ${OpenCV_LIBS})
 37 
 38 link_libraries("/home/huangqp/Coding/deep-learning/ncnn/ncnn/tools/darknet/darknet2ncnn/darknet/libdarknet.so")
 39 add_executable(cifarsmall cifarsmall.cpp)
 40 target_link_libraries(cifarsmall ncnn ${OpenCV_LIBS} libdarknet.so)

进入ncnn/build文件下执行以下命令

$ make clean
$ cmake..
$ make -j4