halcon深度学习-图像分类

halcon深度学习-图像分类

一 训练

*使用深度学习分类功能步骤：
*1.图像分文件夹：在images目录下按类名建文件夹，分别放入对应的图像文件；推荐png或bmp格式，不建议使用jpg格式；
*2.训练：最佳网络存为model_best.hdl，训练结果存图；
*如有问题，联系lidp@daheng-imaging.com
*影响训练效果和时长的参数
*网络类型
Model := 'pretrained_dl_classifier_compact.hdl'
*Model := 'pretrained_dl_classifier_enhanced.hdl'
*Model := 'pretrained_dl_classifier_resnet50.hdl'
*Model := 'model_best.hdl'
*训练迭代次数
NumEpochs := 100*1
*数据扩充比例
AugPercent := 50
*数据扩充是否支持水平、垂直镜像
Mirror := 'rc'
*最大图像宽度
MaxWidth := 672.0*1.0
*显卡序号
GpuId := 0
InitialLearningRate := 0.001
ChangeLearningRateEpochs := int([NumEpochs/4, NumEpochs/2,NumEpochs/4*3])
DecayRate := 0.5
ChangeLearningRateValues := InitialLearningRate * [DecayRate,DecayRate*DecayRate,DecayRate*DecayRate*DecayRate]

ReCreateDataset   := 1
ErrImageTransfer  := 1
TrainingPercent   := 80
ValidationPercent := 100 - TrainingPercent
set_system ('seed_rand', 42)
dev_update_off ()
dev_close_window ()
*LearningRateStepEveryNthEpoch := NumEpochs / 20
*LearningRateStepEveryNthEpoch := 5
*tuple_pow(10, (-2.0 * LearningRateStepEveryNthEpoch / NumEpochs), LearningRateStepRatio)
GenParamName := []
GenParamValue := []
create_dict (AugmentationParam)
set_dict_tuple (AugmentationParam, 'augmentation_percentage', AugPercent)
set_dict_tuple (AugmentationParam, 'mirror', Mirror)
GenParamName := [GenParamName,'augment']
GenParamValue := [GenParamValue,AugmentationParam]
if (|ChangeLearningRateEpochs| > 0)
    create_dict (ChangeStrategy)
    set_dict_tuple (ChangeStrategy, 'model_param', 'learning_rate')
    set_dict_tuple (ChangeStrategy, 'initial_value', InitialLearningRate)
    set_dict_tuple (ChangeStrategy, 'epochs', ChangeLearningRateEpochs)
    set_dict_tuple (ChangeStrategy, 'values', ChangeLearningRateValues)
    GenParamName := [GenParamName,'change']
    GenParamValue := [GenParamValue,ChangeStrategy]
endif
ImageDir := 'images/'
list_files(ImageDir, 'directories', RawImageFolder)
read_dl_dataset_classification (RawImageFolder, 'last_folder', DLDataset)
split_dl_dataset (DLDataset, TrainingPercent, ValidationPercent, [])
read_dl_model (Model, DLModelHandle) 
get_dict_tuple (DLDataset, 'class_names', ClassNames)
set_dl_model_param (DLModelHandle, 'class_names', ClassNames)
get_dict_tuple (DLDataset, 'samples', Samples)
get_dict_tuple (Samples[0], 'image_file_name', ImageFileName)
read_image (Image, ImageDir + '/' + ImageFileName)
if (Model != 'model_best.hdl')
    get_image_size(Image, Width, Height)
    count_channels(Image, Channels)  
    tuple_min2(Width, MaxWidth, Min)
    Scale := Width / Min
    Width  := int (Width  / Scale)
    Height := int (Height / Scale)       
    set_dl_model_param (DLModelHandle, 'image_dimensions', [Width, Height, Channels])    
endif
create_dl_preprocess_param_from_model (DLModelHandle, 'none', 'full_domain', [], [], [], DLPreprocessParam)
create_dict (PreprocessSettings)
set_dict_tuple (PreprocessSettings, 'overwrite_files', true)
preprocess_dl_dataset (DLDataset, 'data', DLPreprocessParam, PreprocessSettings, DLDatasetFileName)
set_dl_model_param (DLModelHandle, 'learning_rate', InitialLearningRate)
set_dl_model_param (DLModelHandle, 'batch_size', 1)
if (GpuId > 0)
    set_dl_model_param (DLModelHandle, 'gpu', GpuId)
endif
tuple_min2(32, |Samples|*TrainingPercent/100/2, BatchSize)
Trying := true
while(Trying)
try
    set_dl_model_param (DLModelHandle, 'batch_size', BatchSize)
    set_dl_model_param (DLModelHandle, 'runtime_init', 'immediately')
    Trying := false
catch (Exception)
    BatchSize:= BatchSize/ 2
    if (BatchSize <= 1)
        Trying := false
    endif
endtry
endwhile
get_dl_model_param (DLModelHandle, 'image_dimensions', ImageDimensions)  
dev_open_window(0, 0, 600, ImageDimensions[1] * 600 / ImageDimensions[0], 'black', WindowHandle)
set_font(WindowHandle, 'default-20')
dev_display (Image)
Text := Model
Text[|Text|] := ImageDimensions[0] + '×' + ImageDimensions[1] + '×' + ImageDimensions[2] + '×' + |Samples|*TrainingPercent/100 + ':' + |Samples|*ValidationPercent/100
Text[|Text|] := 'gpu' + GpuId + ', '  + NumEpochs + ', ' + BatchSize + ', ' + InitialLearningRate
dev_disp_text (Text, 'window', 'top', 'left', 'blue', 'box', 'true')
dump_window(WindowHandle, 'png', 'train_1')
dev_close_window()
create_dl_train_param (DLModelHandle, NumEpochs, 1, 'true', 42, GenParamName, GenParamValue, TrainParam)
train_dl_model (DLDataset, DLModelHandle, TrainParam, 0, TrainResults, TrainInfos, EvaluationInfos)
*保存训练过程曲线
for Index := 2 to 3 by 1
    dev_get_window (WindowHandle)
    dump_window(WindowHandle, 'png', 'train_' + Index)
    dev_close_window()
endfor
* Evaluate
create_dict (GenParamEval)
set_dict_tuple (GenParamEval, 'class_names_to_evaluate', 'global')
set_dict_tuple (GenParamEval, 'measures', ['top1_error','precision','recall','f_score','absolute_confusion_matrix'])
evaluate_dl_model (DLDataset, DLModelHandle, 'split', ['validation','test'], GenParamEval, EvaluationResult, EvalParams)
create_dict (EvalDisplayMode)
set_dict_tuple (EvalDisplayMode, 'display_mode', ['measures','pie_charts_precision','pie_charts_recall','absolute_confusion_matrix'])
create_dict (WindowDict)
dev_display_classification_evaluation (EvaluationResult, EvalParams, EvalDisplayMode, WindowDict)
*保存验证结果图表
get_dict_tuple (WindowDict, 'window_measures', WindowHandle)
dump_window(WindowHandle, 'png', 'measures')
get_dict_tuple (WindowDict, 'window_absolute_confusion_matrix', WindowHandle)
dump_window(WindowHandle, 'png', 'matrix')
get_dict_tuple (WindowDict, 'window_pie_charts_precision', WindowHandle)
dump_window(WindowHandle, 'png', 'precision')
get_dict_tuple (WindowDict, 'window_pie_charts_recall', WindowHandle)
dump_window(WindowHandle, 'png', 'recall')
*显示交互式混淆矩阵
dev_display_dl_interactive_confusion_matrix (DLDataset, EvaluationResult, [])

二 预测

*推理程序：调用model_best.hdl推理test目录下测试图像，结果放在result目录；
*支持显示热度图和网络特征图
TestImageDir := 'test/'
GpuId := 0
ClassifyThreshold := 0.5
dev_update_off()
dev_close_window ()

read_dl_model ('model_best.hdl', DLModelHandle)
set_dl_model_param (DLModelHandle, 'batch_size', 1)
if (GpuId != 0)
    set_dl_model_param (DLModelHandle, 'gpu', GpuId)
endif
set_dl_model_param (DLModelHandle, 'runtime_init', 'immediately')
get_dl_model_param(DLModelHandle, 'image_dimensions', ImageDimensions)
get_dl_model_param(DLModelHandle, 'class_names', ClassNames)
get_dl_model_param(DLModelHandle, 'class_ids', ClassIds)
ResultDir := 'result/'
file_exists(ResultDir, FileExists)
if(FileExists)
    remove_dir_recursively (ResultDir)
endif
make_dir (ResultDir)
for I := 0 to |ClassNames|-1 by 1
    OutputDir:= ResultDir + ClassNames[I] + '/'
    make_dir (OutputDir)
endfor
dev_open_window(0, 0, 600, ImageDimensions[1] * 600 / ImageDimensions[0], 'black', WindowHandle)
set_font(WindowHandle, 'Microsoft YaHei UI-Bold-36')
create_dict (DLSample)
list_image_files (TestImageDir, 'default', 'recursive', ImageFiles)
tics := []
for I := 0 to |ImageFiles|-1 by 1
    read_image (Image, ImageFiles[I])
    count_seconds(Start)
    zoom_image_size (Image, ImagePreprocessedByte, ImageDimensions[0], ImageDimensions[1], 'constant')
    convert_image_type (ImagePreprocessedByte, ImagePreprocessed, 'real')
    scale_image (ImagePreprocessed, ImagePreprocessed, 1, -127)
    set_dict_object (ImagePreprocessed, DLSample, 'image') 
    apply_dl_model (DLModelHandle, DLSample, [], DLResult)
    get_dict_tuple (DLResult, 'classification_confidences', Confidences)
    get_dict_tuple (DLResult, 'classification_class_names', PredictClasses)
    tuple_max (Confidences, Max)
    tuple_find (Confidences, Max, IndexMax)    
    count_seconds(End)
    tics := [tics, (End - Start)*1000]
    dev_clear_window ()
    dev_display(Image)
    Text := ImageFiles[I]
    *Text[|Text|] := PredictClasses[IndexMax] + ', ' + Max$'.2f' + '   ' + (I+1) + '/' + |ImageFiles|
    Text := PredictClasses[IndexMax] + ', ' + Max$'.2f'
    dev_disp_text (Text, 'window', 'top', 'left', 'red', 'box', 'false')    
    if(Max >= ClassifyThreshold)
        parse_filename (ImageFiles[I], BaseName, Extension, Directory) 
        write_image(Image, 'png', 0, ResultDir + PredictClasses[IndexMax] + '/' + BaseName)
        dump_window(WindowHandle, 'png', ResultDir + BaseName)
     endif
endfor
tuple_mean(tics[1:(|tics|-1)], ImageProcTimeMs)
Text := 'infer: ' + ImageDimensions[0] + '×' + ImageDimensions[1] + '×' + ImageDimensions[2] + '×' + |ImageFiles|
Text[|Text|] := 'speed: ' + ImageProcTimeMs$'.1f' + ' ms/image' 
dev_disp_text (Text, 'window', 'center', 'left', 'blue', 'box', 'true')        
dump_window(WindowHandle, 'png', 'speed')
stop()
*显示热度图
create_dict (HeatmapParam)
ChosenLayers := ['res2_block0_conv1']
*set_dict_tuple (HeatmapParam, 'target_conv_layer', ChosenLayers)
*set_dict_tuple (HeatmapParam, 'use_conv_only', 'false')
*set_dict_tuple (HeatmapParam, 'scaling', 'scale_after_relu')
create_dict (DLDatasetInfo)
set_dict_tuple (DLDatasetInfo, 'class_names', ClassNames)
set_dict_tuple (DLDatasetInfo, 'class_ids', ClassIds)
TargetClassID := []
create_dict (WindowHandleDict)
for I := 0 to |ImageFiles|-1 by 1
    read_image (Image, ImageFiles[I])
    zoom_image_size (Image, ImagePreprocessed, ImageDimensions[0], ImageDimensions[1], 'constant')
    convert_image_type (ImagePreprocessed, ImagePreprocessed, 'real')
    scale_image (ImagePreprocessed, ImagePreprocessed, 1, -127)
    set_dict_object (ImagePreprocessed, DLSample, 'image') 
    gen_dl_model_heatmap (DLModelHandle, DLSample, 'grad_cam', TargetClassID, HeatmapParam, DLResult)
    dev_display_dl_data (DLSample, DLResult, DLDatasetInfo, 'heatmap_grad_cam', [], WindowHandleDict)
    get_dict_tuple (WindowHandleDict, 'heatmap_grad_cam', WindowHandle)
    parse_filename (ImageFiles[I], BaseName, Extension, Directory)
    dump_window(WindowHandle[0], 'png', ResultDir + BaseName + '_heatmap')
    *stop()
endfor  
dev_close_window_dict (WindowHandleDict)
stop()
*获取网络特征图
*获取网络结构，包括：ID, NAME, TYPE, OUTPUT_SHAPE, CONNECTED_NODES.
get_dl_model_param(DLModelHandle, 'summary', NetworkSummary)
Depth := |NetworkSummary|
if(Depth == 44)
    ChosenLayers := ['conv1','conv10']
elseif(Depth == 103)
    ChosenLayers := ['convolution_5','convolution_80']      
elseif(Depth == 175)
    ChosenLayers := ['conv1', 'res2_block2_conv1']
endif
get_layer_types_from_summary (ChosenLayers, NetworkSummary, LayerTypes)
create_dict (SelectedFeatureMapDepths)
for LayerIndex := 0 to |ChosenLayers| - 1 by 1
    read_dl_model ('model_best.hdl', DLModelHandle)
    set_dl_model_param (DLModelHandle, 'batch_size', 1)
    set_dl_model_param (DLModelHandle, 'runtime_init', 'immediately')
    set_dl_model_param (DLModelHandle, 'extract_feature_maps', ChosenLayers[LayerIndex])
    set_dict_tuple (SelectedFeatureMapDepths, ChosenLayers[LayerIndex], [1,2,3])   
    for ImageIndex := 0 to 1 by 1
        read_image (Image, ImageFiles[ImageIndex])
        zoom_image_size (Image, ImagePreprocessed, ImageDimensions[0], ImageDimensions[1], 'constant')
        convert_image_type (ImagePreprocessed, ImagePreprocessed, 'real')
        scale_image (ImagePreprocessed, ImagePreprocessed, 1, -127)
        set_dict_object (ImagePreprocessed, DLSample, 'image') 
        apply_dl_model (DLModelHandle, DLSample, [], DLResult)
        get_dict_object (SampleFeatureMaps, DLResult, ChosenLayers[LayerIndex])
        get_feature_maps (SampleFeatureMaps, SelectedFeatureMapDepths, LayerIndex, SelectedFeatureMaps)
        create_dict (WindowDict)
        display_feature_maps (Image, ChosenLayers, LayerIndex, ImageIndex, LayerTypes, SelectedFeatureMaps, SelectedFeatureMapDepths, WindowDict)
        stop ()  
        dev_close_window_dict (WindowDict)
    endfor    
endfor
stop()