halcon例程解析：检查不同药片——check_blister_mixed

原文介绍：

This example demonstrates an application from the pharmaceutical industry. The task is to check the content of manually filled blisters. The first image (reference) shows the combination, that is requested. The subsequent images are then checked for this setup drawn from the reference image. To perform this task we use the gmm classifier, which will be trained with the reference image in the beginning and is then used to classify the different pill types in subsequent blister images.

这个例子演示了一个制药行业的应用程序。任务是检查手工填充的水泡的内容。第一个图像(参考)显示了所请求的组合。随后的图像将被检查为这个从参考图像绘制的设置。为了完成这项任务，我们使用gmm分类器，在开始时会使用参考图像进行训练，然后用于在随后的水泡图像中分类不同类型的药丸。

1. 效果展示

利用一个标准图像做模板，检测药片的有无、错误

模板图与检测图


结果

2. 思路分析

观察模板图像，可知共有三个类别：第一排一组；第二、三排一组；第四、五排一组

这里我们用高斯混合模型来做分类器（GMM）

1. 利用参考图建立药片位置及分类模型
   1. 分割出前景区域（不同于上一篇，这里不需要先对模板区域做矫正）
   2. 确定目标区域，并计算得到其中心与偏转角，用于对检测图像进行刚性矫正
   3. 将三个类别的区域区分出来，得到三个区域，每个区域对应一个类别
   4. 根据三个类别的颜色差别将其区别开，分为三类
   5. 训练高斯混合分类器（GMM）
2. 矫正检测图像，确定检测区域
   1. 分割出前景区域
   2. 计算其旋转、中心
   3. 利用参考图的中心与偏角对检测区域进行刚性变换
3. 对检测区域进行定性分析，分类
   1. 使用训练好的GMM分类器进行分类，分类后得到三个类别分别对应的区域
   2. 逐一取分类后的三个通道之一与模板定好的区域做与，根据面积和宽两个参数分割
   3. 将三个类别结果进行统计，并与模板区域取差，即得到结果
   4. 对小区域进行分析，根据小区域的方差确定是缺失还是错误

2.1 利用参考图建立药片位置及分类模型

* 进行阈值分割，得到前景区域
threshold (Image, Region, 90, 255)
connection (Region, ConnectedRegions)
select_shape (ConnectedRegions, SelectedRegions, 'area', 'and', 5000, 999999)
shape_trans (SelectedRegions, Blister, 'convex')

* 生成药片区域，这个数值是自己判断的
gen_empty_region (Chambers)
for I := 0 to 4 by 1
    Row := 107 + I * 70
    for J := 0 to 2 by 1
        Column := 177 + J * 150
        gen_rectangle2 (Rectangle, Row, Column, 0, 64, 30)
        concat_obj (Chambers, Rectangle, Chambers)
    endfor
endfor
difference (Blister, Chambers, Pattern)
union1 (Chambers, ChambersUnion)

* 计算偏角和中心，这用于对后边待检测图像进行矫正
orientation_region (Blister, PhiRef)
PhiRef := rad(180) + PhiRef
area_center (Blister, Area, RowRef, ColumnRef)
* 
* 
*  +++ Extract pattern for classification +++
* 将模板图像分为三类，首先手动区分，1——145行 145——270行 270——390行
select_shape (Chambers, PillType1, 'row', 'and', 1, 145)
union1 (PillType1, PillType1)
select_shape (Chambers, PillType2, 'row', 'and', 145, 270)
union1 (PillType2, PillType2)
select_shape (Chambers, PillType3, 'row', 'and', 270, 390)
union1 (PillType3, PillType3)
* 
* 
* Extract color space of yellow pills
* 第一类别颜色区分为黄色
reduce_domain (Image, PillType1, ImageReduced)
decompose3 (ImageReduced, ImageR, ImageG, ImageB)
threshold (ImageB, Region, 60, 95)
* 
* Extract color space of red pills
* 第二类别颜色区分为红色
reduce_domain (Image, PillType2, ImageReduced)
decompose3 (ImageReduced, ImageR, ImageG, ImageB)
invert_image (ImageB, ImageInvert)
hysteresis_threshold (ImageInvert, RegionHysteresis2, 190, 200, 5)
* 
* Extract color space of green pills
* 第三类别颜色区分为绿色
reduce_domain (Image, PillType3, ImageReduced)
decompose3 (ImageReduced, ImageR, ImageG, ImageB)
invert_image (ImageB, ImageInvert)
hysteresis_threshold (ImageInvert, RegionHysteresis3, 180, 200, 10)

* 分类三类
intersection (Region, PillType1, PillType1)
intersection (RegionHysteresis2, PillType2, PillType2)
PillTypeCount := [3,6,6]
intersection (RegionHysteresis3, PillType3, PillType3)
concat_obj (PillType1, PillType2, Classes)
concat_obj (Classes, PillType3, Classes)
return ()

最终结果为：

* 训练GMM分类器
NumClasses := |PillTypeCount|
create_class_gmm (3, 3, [1,5], 'spherical', 'normalization', 10, 42, GMMHandle)
add_samples_image_class_gmm (Image, Classes, GMMHandle, 0)
train_class_gmm (GMMHandle, 100, 0.001, 'training', 0.0001, Centers, Iter)

2.2 矫正检测图像，确定检测区域

* 在待检测图像中分割出前景区域
threshold (Image, Region, 90, 255)
connection (Region, ConnectedRegions)
select_shape (ConnectedRegions, SelectedRegions, 'area', 'and', 5000, 9999999)
shape_trans (SelectedRegions, RegionTrans, 'convex')

* 对图像进行仿射变换，向参考图像矫正
orientation_region (RegionTrans, Phi)
if (abs(Phi) > rad(90))
	Phi := rad(180) + Phi
endif
area_center (RegionTrans, Area1, Row, Column)
vector_angle_to_rigid (Row, Column, Phi, RowRef, ColumnRef, PhiRef, HomMat2D)
affine_trans_image (Image, ImageAffineTrans, HomMat2D, 'constant', 'false')

* 取得检测区域
reduce_domain (ImageAffineTrans, ChambersUnion, ImageReduced)
decompose3 (ImageAffineTrans, ImageR, ImageG, ImageB)

2.3 对检测区域进行定性分析，分类

2.3.1 分类

classify_image_class_gmm (ImageReduced, ClassRegions, GMMHandle, 0.0005)

使用训练好的GMM分类器进行分类

2.3.2 分析

count_obj (ClassRegions, Number)
gen_empty_obj (FinalClasses)
connection (Chambers, ChambersRemaining)
* 分类后得到三个类别分别对应的区域
* 逐一取分类后的三个通道之一与模板定好的区域做与，根据面积和宽两个参数分割
for Index := Number to 1 by -1
	dev_clear_window ()
	select_obj (ClassRegions, Region, Index)
	intersection (ChambersRemaining, Region, Region)
	* 根据面积和宽过滤掉不符合的区域
	select_shape (Region, PillsOfOneType, ['area','width'], 'and', [200,40], [3000,80])
	difference (ChambersUnion, PillsOfOneType, RegionDifference)
	connection (RegionDifference, ConnectedRegions)
	select_shape (ConnectedRegions, SelectedRegions, 'area', 'and', 0, 7868)
	shape_trans (SelectedRegions, SelectedRegions, 'convex')
	union1 (SelectedRegions, SelectedRegions)
	
	* 去掉已经检测过的区域
	difference (ChambersRemaining, SelectedRegions, ChambersRemaining)
	concat_obj (SelectedRegions, FinalClasses, FinalClasses)
endfor

2.3.3 利用检测出的错误区域的方差来区分错误和缺失

gen_empty_obj (MissingPills)
gen_empty_obj (WrongPills)
gen_empty_obj (WrongNumberOfPills)
difference (ChambersUnion, FinalClasses, LeftOvers)
area_center (LeftOvers, Area, Row1, Column1)
if (Area > 0)
	connection (LeftOvers, LeftOvers)
	count_obj (LeftOvers, Number)
	for Index := 1 to Number by 1
		select_obj (LeftOvers, ObjectSelected, Index)
		intensity (ObjectSelected, ImageB, Mean, Deviation)
		* 方差>40表示有东西，但不是此类;<40表示缺失
		if (Deviation > 40)
			concat_obj (WrongPills, ObjectSelected, WrongPills)
		else
			concat_obj (MissingPills, ObjectSelected, MissingPills)
		endif
	endfor
endif

3. 全部代码

主代码

* This example demonstrates an application from the pharmaceutical
* industry. The task is to check the content of manually filled
* blisters. The first image (reference) shows the combination, that
* is requested. The subsequent images are then checked for this
* setup drawn from the reference image. To perform this task we use
* the gmm classifier, which will be trained with the reference image
* in the beginning and is then used to classify the different pill
* types in subsequent blister images.
* 
dev_close_window ()
read_image (Image, 'blister/blister_mixed_reference')
dev_open_window_fit_image (Image, 0, 0, -1, -1, WindowHandle)
set_display_font (WindowHandle, 14, 'mono', 'true', 'false')
dev_update_off ()
dev_display (Image)
dev_set_draw ('margin')
* 
* First, we extract the content of the blister and pass
* this information on to the gmm classifier
disp_message (WindowHandle, 'Train gmm classifier on pill types', 'window', 12, 12, 'black', 'true')
extract_pill_types (Image, Chambers, ChambersUnion, Classes, PhiRef, RowRef, ColumnRef, PillTypeCount)
NumClasses := |PillTypeCount|
create_class_gmm (3, 3, [1,5], 'spherical', 'normalization', 10, 42, GMMHandle)
add_samples_image_class_gmm (Image, Classes, GMMHandle, 0)
train_class_gmm (GMMHandle, 100, 0.001, 'training', 0.0001, Centers, Iter)
* 
* 
* Then, the subsequent blisters are tested for their right combination
Count := 12
for FileIndex := 1 to Count by 1
    * 
    * Align image read
    read_image (Image, 'blister/blister_mixed_' + FileIndex$'02')
    threshold (Image, Region, 90, 255)
    connection (Region, ConnectedRegions)
    select_shape (ConnectedRegions, SelectedRegions, 'area', 'and', 5000, 9999999)
    shape_trans (SelectedRegions, RegionTrans, 'convex')
    orientation_region (RegionTrans, Phi)
    if (abs(Phi) > rad(90))
        Phi := rad(180) + Phi
    endif
    area_center (RegionTrans, Area1, Row, Column)
    vector_angle_to_rigid (Row, Column, Phi, RowRef, ColumnRef, PhiRef, HomMat2D)
    affine_trans_image (Image, ImageAffineTrans, HomMat2D, 'constant', 'false')
    reduce_domain (ImageAffineTrans, ChambersUnion, ImageReduced)
    decompose3 (ImageAffineTrans, ImageR, ImageG, ImageB)
    * 
    * Classify pill type for each chamber
    classify_image_class_gmm (ImageReduced, ClassRegions, GMMHandle, 0.0005)
    * 
    count_obj (ClassRegions, Number)
    gen_empty_obj (FinalClasses)
    connection (Chambers, ChambersRemaining)
    for Index := Number to 1 by -1
        dev_clear_window ()
        select_obj (ClassRegions, Region, Index)
        intersection (ChambersRemaining, Region, Region)
        select_shape (Region, PillsOfOneType, ['area','width'], 'and', [200,40], [3000,80])
        difference (ChambersUnion, PillsOfOneType, RegionDifference)
        connection (RegionDifference, ConnectedRegions)
        select_shape (ConnectedRegions, SelectedRegions, 'area', 'and', 0, 7868)
        shape_trans (SelectedRegions, SelectedRegions, 'convex')
        union1 (SelectedRegions, SelectedRegions)
        difference (ChambersRemaining, SelectedRegions, ChambersRemaining)
        concat_obj (SelectedRegions, FinalClasses, FinalClasses)
    endfor
    * 
    * 
    * Check for right combination
    gen_empty_obj (MissingPills)
    gen_empty_obj (WrongPills)
    gen_empty_obj (WrongNumberOfPills)
    difference (ChambersUnion, FinalClasses, LeftOvers)
    area_center (LeftOvers, Area, Row1, Column1)
    if (Area > 0)
        connection (LeftOvers, LeftOvers)
        count_obj (LeftOvers, Number)
        for Index := 1 to Number by 1
            select_obj (LeftOvers, ObjectSelected, Index)
            intensity (ObjectSelected, ImageB, Mean, Deviation)
            if (Deviation > 40)
                concat_obj (WrongPills, ObjectSelected, WrongPills)
            else
                concat_obj (MissingPills, ObjectSelected, MissingPills)
            endif
        endfor
    endif
    * 
    * Compute histogram
    CountFinalClass := []
    for Index := 1 to NumClasses by 1
        select_obj (FinalClasses, ObjectSelected, Index)
        connection (ObjectSelected, ObjectSelected)
        count_obj (ObjectSelected, Size)
        CountFinalClass := [CountFinalClass,Size]
    endfor
    * 
    * Display classification results and output allover statistic
    display_results (ImageAffineTrans, LeftOvers, FinalClasses, WrongPills, CountFinalClass, PillTypeCount, WindowHandle)
    if (FileIndex < Count)
        disp_continue_message (WindowHandle, 'black', 'true')
    endif
    stop ()
endfor

extract_pill_types

threshold (Image, Region, 90, 255)
connection (Region, ConnectedRegions)
select_shape (ConnectedRegions, SelectedRegions, 'area', 'and', 5000, 999999)
shape_trans (SelectedRegions, Blister, 'convex')
gen_empty_region (Chambers)
for I := 0 to 4 by 1
    Row := 107 + I * 70
    for J := 0 to 2 by 1
        Column := 177 + J * 150
        gen_rectangle2 (Rectangle, Row, Column, 0, 64, 30)
        concat_obj (Chambers, Rectangle, Chambers)
    endfor
endfor
difference (Blister, Chambers, Pattern)
union1 (Chambers, ChambersUnion)
orientation_region (Blister, PhiRef)
PhiRef := rad(180) + PhiRef
area_center (Blister, Area, RowRef, ColumnRef)
* 
* 
*  +++ Extract pattern for classification +++
select_shape (Chambers, PillType1, 'row', 'and', 1, 145)
union1 (PillType1, PillType1)
select_shape (Chambers, PillType2, 'row', 'and', 145, 270)
union1 (PillType2, PillType2)
select_shape (Chambers, PillType3, 'row', 'and', 270, 390)
union1 (PillType3, PillType3)
* 
* 
* Extract color space of yellow pills
reduce_domain (Image, PillType1, ImageReduced)
decompose3 (ImageReduced, ImageR, ImageG, ImageB)
threshold (ImageB, Region, 60, 95)
* 
* Extract color space of red pills
reduce_domain (Image, PillType2, ImageReduced)
decompose3 (ImageReduced, ImageR, ImageG, ImageB)
invert_image (ImageB, ImageInvert)
hysteresis_threshold (ImageInvert, RegionHysteresis2, 190, 200, 5)
* 
* Extract color space of green pills
reduce_domain (Image, PillType3, ImageReduced)
decompose3 (ImageReduced, ImageR, ImageG, ImageB)
invert_image (ImageB, ImageInvert)
hysteresis_threshold (ImageInvert, RegionHysteresis3, 180, 200, 10)
* 
intersection (Region, PillType1, PillType1)
intersection (RegionHysteresis2, PillType2, PillType2)
PillTypeCount := [3,6,6]
intersection (RegionHysteresis3, PillType3, PillType3)
concat_obj (PillType1, PillType2, Classes)
concat_obj (Classes, PillType3, Classes)
return ()