The Setup of Deeplab v2

1. Clone the code

Refer to http://blog.csdn.net/Xmo_jiao/article/details/77897109

2. Make the dataset

Refer to https://blog.csdn.net/u014451076/article/details/79700653
Note: When we convert the ground truth from RGB to Gray, the original train and val label of VOC should keep the white boundary which will influence the result(i.e higher mIoU).

3. In the process of compiling the caffe-deeplab-v2

We need to compile caffe-deeplab-v2.
Because the cudnn version used by the auther of deeplab v2 is 4.0, Usually the version we used is >=5.0 and we need to modify this code.
Refer to http://blog.csdn.net/tianrolin/article/details/71246472

4. Train

Refer to http://blog.csdn.net/Xmo_jiao/article/details/77897109
The basic procedure is:

• modify some paths in run_pascal.sh
• modify train_aug.txt and val.txt in direcotary list
• run run_pascal.sh to train
• run run_pascal.sh to test, the result is .mat
• run deeplabv2/matlab/EvalSegResult.m to evaluate

5. Evaluate(deeplab-vgg16-MS-LargeFOV)

Modify deeplab-public-ver2/matlab/my_script/EvalSegResults.m to test. One example is below:

SetupEnv;

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% You do not need to chage values below
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

if is_server
  if strcmp(dataset, 'voc12')
    VOC_root_folder = '/path/todata/pascal';
  elseif strcmp(dataset, 'coco')
    VOC_root_folder = '/rmt/data/coco';
  else
    error('Wrong dataset');
  end
else
  if strcmp(dataset, 'voc12')  
    VOC_root_folder = '~/dataset/PASCAL/VOCdevkit';
  elseif strcmp(dataset, 'coco')
    VOC_root_folder = '~/dataset/coco';
  else
    error('Wrong dataset');
  end
end

if has_postprocess
  if learn_crf
    post_folder = sprintf('post_densecrf_W%d_XStd%d_RStd%d_PosW%d_PosXStd%d_ModelType%d_Epoch%d', bi_w, bi_x_std, bi_r_std, pos_w, pos_x_std, model_type, epoch); 
  else
    post_folder = sprintf('post_densecrf_W%d_XStd%d_RStd%d_PosW%d_PosXStd%d', bi_w, bi_x_std, bi_r_std, pos_w, pos_x_std); 
  end
else
  post_folder = 'post_none';
end

%output_mat_folder = fullfile('/path/to/deeplab_v2/voc2012', feature_name, model_name, testset, feature_type);
output_mat_folder = '/path/to/deeplab_v2/voc2012/features/deeplab_largeFOV/val/fc8';

save_root_folder = fullfile('/path/to/deeplab_v2/voc2012', 'res', feature_name, model_name, testset, feature_type, post_folder);

fprintf(1, 'Saving to %s\n', save_root_folder);

if strcmp(dataset, 'voc12')
  seg_res_dir = [save_root_folder '/results/VOC2012/'];
  seg_root = fullfile(VOC_root_folder, 'VOC2012');
  gt_dir   = fullfile(VOC_root_folder, 'VOC2012', 'SegmentationClass');
elseif strcmp(dataset, 'coco')
  seg_res_dir = [save_root_folder '/results/COCO2014/'];
  seg_root = fullfile(VOC_root_folder, '');
  gt_dir   = fullfile(VOC_root_folder, '', 'SegmentationClass');
end

save_result_folder = fullfile(seg_res_dir, 'Segmentation', [id '_' testset '_cls']);

if ~exist(save_result_folder, 'dir')
    mkdir(save_result_folder);
end

if strcmp(dataset, 'voc12')
  VOCopts = GetVOCopts(seg_root, seg_res_dir, trainset, testset, 'VOC2012');
elseif strcmp(dataset, 'coco')
  VOCopts = GetVOCopts(seg_root, seg_res_dir, trainset, testset, '');
end

if is_mat
  % crop the results
  load('pascal_seg_colormap.mat');

  output_dir = dir(fullfile(output_mat_folder, '*.mat'));

  for i = 1 : numel(output_dir)
    if mod(i, 100) == 0
        fprintf(1, 'processing %d (%d)...\n', i, numel(output_dir));
    end

    data = load(fullfile(output_mat_folder, output_dir(i).name));
    % load .mat
    raw_result = data.data;
    raw_result = permute(raw_result, [2 1 3]);

    img_fn = output_dir(i).name(1:end-4);
    img_fn = strrep(img_fn, '_blob_0', '');

    if strcmp(dataset, 'voc12')
      img = imread(fullfile(VOC_root_folder, 'VOC2012', 'JPEGImages', [img_fn, '.jpg']));
    elseif strcmp(dataset, 'coco')
      img = imread(fullfile(VOC_root_folder, 'JPEGImages', [img_fn, '.jpg']));
    end

    % get the original size from the original image so as to clip
    img_row = size(img, 1);
    img_col = size(img, 2);

    % clip
    result = raw_result(1:img_row, 1:img_col, :);

    if ~is_argmax
      [~, result] = max(result, [], 3);
      result = uint8(result) - 1;
    else
      result = uint8(result);
    end

    if debug
        gt = imread(fullfile(gt_dir, [img_fn, '.png']));
        figure(1), 
        subplot(221),imshow(img), title('img');
        subplot(222),imshow(gt, colormap), title('gt');
        subplot(224), imshow(result,colormap), title('predict');
    end

    imwrite(result, colormap, fullfile(save_result_folder, [img_fn, '.png']));
  end
end

% get iou score
if strcmp(testset, 'val')
  [accuracies, avacc, conf, rawcounts] = MyVOCevalseg(VOCopts, id);
else
  fprintf(1, 'This is test set. No evaluation. Just saved as png\n');
end 

result is below(with out CRF).

processing 100 (1449)...
processing 200 (1449)...
processing 300 (1449)...
processing 400 (1449)...
processing 500 (1449)...
processing 600 (1449)...
processing 700 (1449)...
processing 800 (1449)...
processing 900 (1449)...
processing 1000 (1449)...
processing 1100 (1449)...
processing 1200 (1449)...
processing 1300 (1449)...
processing 1400 (1449)...
test confusion: 112/1449
test confusion: 237/1449
test confusion: 341/1449
test confusion: 458/1449
test confusion: 576/1449
test confusion: 700/1449
test confusion: 806/1449
test confusion: 912/1449
test confusion: 1022/1449
test confusion: 1127/1449
test confusion: 1236/1449
test confusion: 1348/1449
Percentage of pixels correctly labelled overall: 92.531%
Accuracy for each class (pixel accuracy)
      background: 96.219%
       aeroplane: 93.273%
         bicycle: 81.932%
            bird: 88.940%
            boat: 83.684%
          bottle: 81.487%
             bus: 90.519%
             car: 89.931%
             cat: 93.166%
           chair: 49.704%
             cow: 79.759%
     diningtable: 57.427%
             dog: 86.444%
           horse: 85.098%
       motorbike: 86.893%
          person: 87.983%
     pottedplant: 63.491%
           sheep: 82.238%
            sofa: 52.055%
           train: 85.916%
       tvmonitor: 78.807%
-------------------------
Mean Class Accuracy: 80.713%
Accuracy for each class (intersection/union measure)
      background: 92.067%
       aeroplane: 82.498%
         bicycle: 36.758%
            bird: 79.323%
            boat: 63.057%
          bottle: 69.035%
             bus: 86.545%
             car: 80.722%
             cat: 82.480%
           chair: 34.429%
             cow: 71.885%
     diningtable: 51.089%
             dog: 76.410%
           horse: 71.342%
       motorbike: 74.273%
          person: 79.867%
     pottedplant: 49.074%
           sheep: 75.517%
            sofa: 43.021%
           train: 79.163%
       tvmonitor: 64.390%
-------------------------
Average accuracy: 68.712%

6. Run_densecrf.py

• firstly, make deeplab_v2/densecrf
  Maybe, we will happen to this error

collect2: error: ld returned 1 exit status
make[1]: *** [prog_refine_pascal_v4] Error 1
make[1]: Leaving directory `/path/to/deeplab_v2/deeplab-public-ver2/densecrf'
make: *** [all] Error 2

To solve this problem, we modify the Makefile by line 31,

$(CC) refine_pascal_v4/dense_inference.cpp -o prog_refine_pascal_v4     $(CFLAGS) -L. -lDenseCRF -lmatio -lhdf5 -I./util/

Then, make successful

• Use deeplab_v2/densecrf/my_script/SaveJpgToPPM.m to transfer JPEGImages to PPMImages
• run deeplab_v2/voc2012/run_densecrf.py to get the result of post-processing, but the format is bin. One example of run_densecrf.py is below.

#!/bin/bash 

###########################################
# You can either use this script to generate the DenseCRF post-processed results
# or use the densecrf_layer (wrapper) in Caffe
###########################################
DATASET=voc12
LOAD_MAT_FILE=1

MODEL_NAME=deeplab_largeFOV

TEST_SET=val           #val, test

# the features  folder save the features computed via the model trained with the train set
# the features2 folder save the features computed via the model trained with the trainval set
FEATURE_NAME=features #features, features2
FEATURE_TYPE=fc8

# specify the parameters
MAX_ITER=10

Bi_W=4
Bi_X_STD=49
Bi_Y_STD=49
Bi_R_STD=5
Bi_G_STD=5 
Bi_B_STD=5

POS_W=3
POS_X_STD=3
POS_Y_STD=3


#######################################
# MODIFY THE PATY FOR YOUR SETTING
#######################################
SAVE_DIR=/path/to/deeplab_v2/voc2012/res/${FEATURE_NAME}/${MODEL_NAME}/${TEST_SET}/${FEATURE_TYPE}/post_densecrf_W${Bi_W}_XStd${Bi_X_STD}_RStd${Bi_R_STD}_PosW${POS_W}_PosXStd${POS_X_STD}

echo "SAVE TO ${SAVE_DIR}"

CRF_DIR=/path/to/deeplab_v2/deeplab-public-ver2/densecrf

if [ ${DATASET} == "voc12" ]
then
    IMG_DIR_NAME=pascal/VOC2012
elif [ ${DATASET} == "coco" ]
then
    IMG_DIR_NAME=coco
elif [ ${DATASET} == "voc10_part" ]
then
    IMG_DIR_NAME=pascal/VOCdevkit/VOC2012
fi

# NOTE THAT the densecrf code only loads ppm images
IMG_DIR=/path/to/data/${IMG_DIR_NAME}/PPMImages

if [ ${LOAD_MAT_FILE} == 1 ]
then
    # the features are saved in .mat format
    CRF_BIN=${CRF_DIR}/prog_refine_pascal_v4
    FEATURE_DIR=/path/to/deeplab_v2/voc2012/${FEATURE_NAME}/${MODEL_NAME}/${TEST_SET}/${FEATURE_TYPE}
else
    # the features are saved in .bin format (has called SaveMatAsBin.m in the densecrf/my_script)
    CRF_BIN=${CRF_DIR}/prog_refine_pascal
    FEATURE_DIR=/rmt/work/deeplab/exper/${DATASET}/${FEATURE_NAME}/${MODEL_NAME}/${TEST_SET}/${FEATURE_TYPE}/bin
fi

mkdir -p ${SAVE_DIR}

#echo ${CRF_BIN} -id ${IMG_DIR} -fd ${FEATURE_DIR} -sd ${SAVE_DIR} -i ${MAX_ITER} -px ${POS_X_STD} -py ${POS_Y_STD} -    pw ${POS_W} -bx ${Bi_X_STD} -by ${Bi_Y_STD} -br ${Bi_R_STD} -bg ${Bi_G_STD} -bb ${Bi_B_STD} -bw ${Bi_W}

# run the program
${CRF_BIN} -id ${IMG_DIR} -fd ${FEATURE_DIR} -sd ${SAVE_DIR} -i ${MAX_ITER} -px ${POS_X_STD} -py ${POS_Y_STD} -pw ${POS_W} -bx ${Bi_X_STD} -by ${Bi_Y_STD} -br ${Bi_R_STD} -bg ${Bi_G_STD} -bb ${Bi_B_STD} -bw ${Bi_W}

• Evaluate the result after CRF(deeplab-vgg16-MS-LargeFOV)

pixel accuracy:  0.933649579242
mean accuracy:  0.811556028833
mIoU:  0.712235214969
fw IU:  0.880878180695