Halcon学习--深度学习（3）-classify_evaluation

* 
* This example is part of a series of examples, which summarizes
* the workflow for DL classification. It uses the MVTec pill dataset.
* The four parts are:
* 1. Dataset preprocessing
* 2. Training of the model.
* 3. Evaluation of the trained model.
* 4. Inference on new images.
* 
* This examples contains part 3: 'Evaluation of the trained model'.
* 
* Please note: This example requires the output of part 1:
* classify_pill_defects_deep_learning_1_preprocess.hdev.
* If you set UsePretrainedModel := false, the output of part 2,
* classify_pill_defects_deep_learning_2_train.hdev,
* is also required.
* 
dev_update_off ()
* 
* In this example, the evaluation is explained in graphics windows,
* before they are executed. Set the following parameter to false in order to
* skip this visualization.
ShowExampleScreens := false
* 
* By default, this example uses a model pretrained by MVTec. To use the model
* which was trained in part 2 of this example series, set the following
* variable to false.
UsePretrainedModel := false
* 
* The evaluation can be performed on GPU or CPU.
* See the respective system requirements in the Installation Guide.
* If possible a GPU is used in this example.
* In case you explicitely wish to run this example on the CPU,
* choose the CPU device instead.
query_available_dl_devices (['runtime','runtime'], ['gpu','cpu'], DLDeviceHandles)
if (|DLDeviceHandles| == 0)
    throw ('No supported device found to continue this example.')
endif
* Due to the filter used in query_available_dl_devices, the first device is a GPU, if available.
DLDevice := DLDeviceHandles[0]
* 
if (ShowExampleScreens)
    * Initial example windows and parameters etc.
    dev_example_init (ShowExampleScreens, UsePretrainedModel, ExampleInternals)
    * 
    * Example series introduction text.
    dev_display_screen_introduction (ExampleInternals)
    stop ()
    * 
    * Evaluation measures.
    dev_display_screen_evaluation_measures (ExampleInternals)
    stop ()
    dev_display_screen_topK (ExampleInternals)
    stop ()
    dev_display_screen_confusion_matrix (ExampleInternals)
    stop ()
    dev_display_screen_precision_recall_f1score (ExampleInternals)
    stop ()
    * 
    * Visual inspection.
    dev_display_screen_visual_inspection_intro (ExampleInternals)
    stop ()
    * 
    * Model optimization.
    dev_display_screen_model_optimization_info (ExampleInternals)
    stop ()
    * 
    * Mention on which device the deep learning operators will run.
    dev_display_screen_device (ExampleInternals, DLDevice)
    stop ()
    * 
    * Start the program.
    dev_display_screen_run_program (ExampleInternals)
    stop ()
    * 
    * Terminate example screens.
    dev_display_example_reset_windows (ExampleInternals)
endif
* 
* 
* ******************************************************
* **   Set paths and parameters for the evaluation   ***
* ******************************************************
* 
* Paths.
* 
* Project directory for any outputs written by HALCON.
ExampleDataDir := 'classify_pill_defects_data'
* File path of the preprocessed DLDataset.
* Note: Adapt DataDirectory after preprocessing with another image size.
DataDirectory := ExampleDataDir + '/dldataset_pill_300x300'
DLDatasetFileName := DataDirectory + '/dl_dataset.hdict'
* 
if (UsePretrainedModel)
    * Use the pretrained model shipping with HALCON.
    RetrainedModelFileName := 'classify_pill_defects.hdl'
else
    * Path of the retrained classification model.
    RetrainedModelFileName := ExampleDataDir + '/best_dl_model_classification.hdl'
endif
* 
* Evaluation parameters.
* 
* Evaluation measures.
ClassificationMeasures := ['top1_error','precision','recall','f_score','absolute_confusion_matrix','relative_confusion_matrix']
* Batch size used during evaluation.
BatchSize := 10
* 
* **********************************
* **   Evaluation of the model   ***
* **********************************
* 
* Check if all necessary files exist.
check_data_availability (ExampleDataDir, DLDatasetFileName, RetrainedModelFileName, UsePretrainedModel)
* 
* Read the retrained model.
read_dl_model (RetrainedModelFileName, DLModelHandle)
* 
set_dl_model_param (DLModelHandle, 'batch_size', BatchSize)
* 
set_dl_model_param (DLModelHandle, 'device', DLDevice)
* 
* Read the preprocessed DLDataset file.
read_dict (DLDatasetFileName, [], [], DLDataset)
* 
* Set parameters for evaluation.
create_dict (GenParamEval)
set_dict_tuple (GenParamEval, 'measures', ClassificationMeasures)
set_dict_tuple (GenParamEval, 'show_progress', 'true')
* 
* Evaluate the retrained model.
evaluate_dl_model (DLDataset, DLModelHandle, 'split', 'test', GenParamEval, EvaluationResult, EvalParams)
* 
* 
* ******************************
* **   Display the results   ***
* ******************************
* 
* Display measures.
create_dict (WindowHandleDict)
create_dict (GenParamEvalDisplay)
set_dict_tuple (GenParamEvalDisplay, 'display_mode', ['measures','pie_charts_precision','pie_charts_recall','absolute_confusion_matrix'])
dev_display_classification_evaluation (EvaluationResult, EvalParams, GenParamEvalDisplay, WindowHandleDict)
dev_disp_text ('Press F5 to continue', 'window', 'bottom', 'right', 'black', [], [])
* 
stop ()
dev_close_window_dict (WindowHandleDict)
* 
* Call interactive confusion matrix.
*dev_display_dl_interactive_confusion_matrix (DLDataset, EvaluationResult, [])
* 
* Close window handles.
dev_close_window_dict (WindowHandleDict)
* 
* 
* **************************************
* **   Visual inspection of images   ***
* **************************************
* 
* To inspect some examples more precisely,
* calculate and display a heatmap.
* Here, we choose the samples
* labeled and classified as 'contamination'.
* 
SelectedHeatmapGTClassName := 'contamination'
SelectedHeatmapInfClassName := 'contamination'
* 
* Get information from DLDataset and EvaluationResult.
get_dict_tuple (EvaluationResult, 'evaluated_samples', EvaluatedSamples)
get_dict_tuple (EvaluatedSamples, 'image_ids', ImageIDs)
get_dict_tuple (EvaluatedSamples, 'image_label_ids', ImageLabelIDs)
get_dict_tuple (EvaluatedSamples, 'top1_predictions', Predictions)
get_dict_tuple (DLDataset, 'class_names', ClassNames)
get_dict_tuple (DLDataset, 'class_ids', ClassIDs)
* Get class IDs for selected classes.
PredictedClassID := ClassIDs[find(ClassNames,SelectedHeatmapInfClassName)]
GroundTruthClassID := ClassIDs[find(ClassNames,SelectedHeatmapGTClassName)]
* Get tuple position of selected classes.
GTIndices := find(ImageLabelIDs [==] GroundTruthClassID,1)
PredictionIndices := find(Predictions [==] PredictedClassID,1)
* Get image IDs for selected combination.
ImageIDsSelected := []
if (PredictionIndices != -1 and PredictionIndices != [])
    ImageIDsSelected := ImageIDs[intersection(GTIndices,PredictionIndices)]
endif
* 
* We offer two heatmap options:
* 1) a fast heatmap operator supporting the method 'grad_cam'
* 2) a confidence-based approach implemented as procedure.
* In this example, set HeatmapMethod to 'heatmap_grad_cam'
* or 'heatmap_confidence_based' to switch between the heatmap options.
HeatmapMethod := 'heatmap_grad_cam'
* 
* Set the target class ID or [] to show the heatmap
* for the classified class.
TargetClassID := []
create_dict (HeatmapParam)
if (HeatmapMethod == 'heatmap_grad_cam')
    * Set generic parameters for operator heatmap.
    set_dict_tuple (HeatmapParam, 'use_conv_only', 'false')
    set_dict_tuple (HeatmapParam, 'scaling', 'scale_after_relu')
elseif (HeatmapMethod == 'heatmap_confidence_based')
    * Set target class ID.
    set_dict_tuple (HeatmapParam, 'target_class_id', TargetClassID)
    * Set the feature size and the sampling size for the
    * confidence based approach.
    FeatureSize := 30
    SamplingSize := 10
    set_dict_tuple (HeatmapParam, 'feature_size', FeatureSize)
    set_dict_tuple (HeatmapParam, 'sampling_size', SamplingSize)
else
    throw ('Unsupported heatmap method.')
endif
* 
* Heatmaps are displayed in sequence, hence set batch size to 1.
set_dl_model_param (DLModelHandle, 'batch_size', 1)
* 
* Visualize heatmaps for selected samples.
create_dict (WindowHandleDict)
get_dict_tuple (DLDataset, 'samples', DatasetSamples)
for Index := 0 to min([|ImageIDsSelected| - 1,10]) by 1
    * Select the corresponding DLSample.
    find_dl_samples (DatasetSamples, 'image_id', ImageIDsSelected[Index], 'match', DLSampleIndex)
    read_dl_samples (DLDataset, DLSampleIndex, DLSample)
    * 
    if (HeatmapMethod == 'heatmap_grad_cam')
        gen_dl_model_heatmap (DLModelHandle, DLSample, 'grad_cam', TargetClassID, HeatmapParam, DLResult)
    else
        * Create temporary DLResult for display.
        create_dict (DLResult)
        gen_dl_model_classification_heatmap (DLModelHandle, DLSample, DLResult, HeatmapParam)
    endif
    dev_display_dl_data (DLSample, DLResult, DLDataset, HeatmapMethod, [], WindowHandleDict)
    dev_disp_text ('Press F5 to continue.', 'window', 'bottom', 'right', 'black', [], [])
    stop ()
endfor
* 
* Optimize the memory consumption.
set_dl_model_param (DLModelHandle, 'optimize_for_inference', 'true')
write_dl_model (DLModelHandle, RetrainedModelFileName)
* Close the windows.
dev_close_window_dict (WindowHandleDict)
* 
if (ShowExampleScreens)
    * Display final screen.
    dev_display_screen_final (ExampleInternals)
    stop ()
    * Close example windows.
    dev_close_example_windows (ExampleInternals)
endif