信息熵信息增益计算Python编程
在周志华《机器学习》书中,第四章讲了关于决策树的相关知识,其中包括信息熵、信息增益的公式。并且给出了书中的数据的信息熵,信息增益的值。由于有些好奇这些数据是否足够正确,并且也希望能够理解信息熵、信息增益的计算过程以及方式,所以编写了如下的程序。另一个原因也是手动计算,过于麻烦。
import numpy as np
import pandas as pd
# load the csv file as a pandas.core.frame.DataFrame
dataset = pd.read_csv('watermelon.csv', header =None, names=['id', 'color', 'root', 'knock', 'veins', 'navel', 'touch', 'class'])
X = dataset.iloc[:, 1:7]
y = dataset.iloc[:, 7].values
def cal_information_entropy(list1):
'''
计算单个信息熵
:param list1: 是一个比值的列表
'''
Ent = 0.0
for x in list1:
Ent += x * np.log2(x)
return -Ent
def cal_information_gain(Ent_D, list_Ent_Di, list_Di, D):
'''
计算信息增益
:param Ent_D: 集合D的信息熵
:param list_Ent_Di:
:param list_Di:
'''
sum_Ent_Di = 0.0
for i in range(len(list_Di)):
sum_Ent_Di = abs(list_Di[i]) / abs(D) * list_Ent_Di[i]
Gain = Ent_D -sum_Ent_Di
return Gain
total = len(y)
good_watermelon = list(y).count(1) # list(y) y的类型不变 统计表中好瓜个数
bad_watermelon = list(y).count(0) # 统计坏瓜个数
kind = [good_watermelon / total, bad_watermelon / total] # 好瓜与坏瓜所占比例
Ent_D = cal_information_entropy(kind)
Ent_D = round(Ent_D, 3) # 信息熵 保留三位小数
# 将所有属性存储到一个字典中
watermelon_attr = {}
watermelon_attr['color'] = X.iloc[:, 0].values
watermelon_attr['root'] = X.iloc[:, 1].values
watermelon_attr['knock'] = X.iloc[:, 2].values
watermelon_attr['veins'] = X.iloc[:, 3].values
watermelon_attr['navel'] = X.iloc[:, 4].values
watermelon_attr['touch'] = X.iloc[:, 5].values
print(watermelon_attr)
# 数据划分 划分为好瓜和坏瓜,并记录好坏瓜的编号
good_watermelon_id = []
bad_watermelon_id = []
for i in range(0, len(y)):
if y[i] == 1:
good_watermelon_id.append(i)
else:
bad_watermelon_id.append(i)
# 对数据按照属性值进行划分计数
# 最终结果存储于字典data_partition中,
# 其形式为{'color': {1: 6, 2: 6, 3: 5}, 'root': {1: 8, 2: 7, 3: 2}, 'knock': {1: 10, 2: 5, 3: 2}, 'veins': {1: 9, 2: 5, 3: 3}, 'navel': {1: 7, 2: 6, 3: 4}, 'touch': {1: 12, 2: 5}}
def partiton(watermelons):
data_partition = {}
for x in watermelons.columns:
data_partition[x] = {}
varlist = watermelons[x].values
for i in varlist:
if i in data_partition[x]:
data_partition[x][i] += 1
else:
data_partition[x][i] = 1
return data_partition
# 计算每个属性值,占该属性的比例
# {'color': {1: 0.5, 2: 0.6666666666666666, 3: 0.2}, 'root': {1: 0.625, 2: 0.42857142857142855}, 'knock': {1: 0.6, 2: 0.4}, 'veins': {1: 0.7777777777777778, 2: 0.2}, 'navel': {1: 0.7142857142857143, 2: 0.5}, 'touch': {1: 0.5, 2: 0.4}}
# 这里,数据被分为好瓜和坏瓜两种属性,所以会分别对好瓜和坏瓜计算比例
def cal_pk(watermelon_partition, watermelon):
watermelon_p = {}
for keys1, values1 in watermelon_partition.items():
watermelon_p[keys1] = {}
for i_keys1, i_values1 in values1.items():
watermelon_p[keys1][i_keys1] = i_values1 / watermelon[keys1][i_keys1]
return watermelon_p
def information_entropy(watermelon, g_w_p, b_w_p):
'''
计算信息熵
:param watermelon: 西瓜数据
:param g_w_p: 好瓜中各属性,其属性值占本属性的比例
:param b_w_p: 坏瓜中各属性,其属性值占本属性的比例
'''
Ent_dic = {}
for keys, values in watermelon.items():
Ent_dic[keys] = []
for i_keys, i_values in values.items():
try:
a = g_w_p[keys][i_keys]
except KeyError:
a = 0
try:
b = b_w_p[keys][i_keys]
except KeyError:
b = 0
if a == 0:
ent = -(b * np.log2(b))
else:
ent = - (a * np.log2(a) + b * np.log2(b))
Ent_dic[keys].append(ent)
return Ent_dic
# 对每个属性进行划分
X_good_watermelon = X[y == 1]
good_watermelon_s = len(good_watermelon_id)
X_bad_watermelon = X[y == 0]
bad_watermelon_s = len(bad_watermelon_id)
good_watermelon_partition = partiton(X_good_watermelon)
bad_watermelon_partition = partiton(X_bad_watermelon)
X_watermelon = partiton(X)
good_watermelon_p = cal_pk(good_watermelon_partition, X_watermelon)
bad_watermelon_p = cal_pk(bad_watermelon_partition, X_watermelon)
Ent_dic = information_entropy(X_watermelon, good_watermelon_p, bad_watermelon_p)
Gain_dic = {}
for keys, values in X_watermelon.items():
listtemp = Ent_dic[keys]
Gain_temp = 0.0
for i_keys, i_values in values.items():
Gain_temp += (X_watermelon[keys][i_keys] / total) *listtemp[int(i_keys)-1]
Gain_temp = Ent_D - Gain_temp
Gain_dic[keys] = Gain_temp
print(total)
print(X_watermelon)
print(Gain_dic)
最终的计算结果为:
watermelon_attr: {'color': array([1, 2, 2, 1, 3, 1, 2, 2, 2, 1, 3, 3, 1, 3, 2, 3, 1], dtype=int64), 'root': array([1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 1, 2, 2, 2, 1, 1], dtype=int64), 'knock': array([1, 2, 1, 2, 1, 1, 1, 1, 2, 3, 3, 1, 1, 2, 1, 1, 2], dtype=int64), 'veins': array([1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 3, 3, 2, 2, 1, 3, 2], dtype=int64), 'navel': array([1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 1, 1, 2, 3, 2], dtype=int64), 'touch': array([1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 2, 1, 1, 2, 1, 1], dtype=int64)}
total: 17
X_watermelon: {'color': {1: 6, 2: 6, 3: 5}, 'root': {1: 8, 2: 7, 3: 2}, 'knock': {1: 10, 2: 5, 3: 2}, 'veins': {1: 9, 2: 5, 3: 3}, 'navel': {1: 7, 2: 6, 3: 4}, 'touch': {1: 12, 2: 5}}
Gain_dic: {'color': 0.10862261889625002, 'root': 0.1431724131976776, 'knock': 0.14127888724588056, 'veins': 0.3810893509991533, 'navel': 0.28965623647256367, 'touch': 0.006543942807450298}
要注意,这里的数据保存为.csv的格式
这里,我给出我的编码规则。
编号 | 色泽 | 根蒂 | 敲声 | 纹理 | 脐部 | 触感 | 好瓜 |
---|---|---|---|---|---|---|---|
1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 |
2 | 2 | 1 | 2 | 1 | 1 | 1 | 1 |
3 | 2 | 1 | 1 | 1 | 1 | 1 | 1 |
4 | 1 | 1 | 2 | 1 | 1 | 1 | 1 |
5 | 3 | 1 | 1 | 1 | 1 | 1 | 1 |
6 | 1 | 2 | 1 | 1 | 2 | 2 | 1 |
7 | 2 | 2 | 1 | 2 | 2 | 2 | 1 |
8 | 2 | 2 | 1 | 1 | 2 | 1 | 1 |
9 | 2 | 2 | 2 | 2 | 2 | 1 | 0 |
10 | 1 | 3 | 3 | 1 | 3 | 2 | 0 |
11 | 3 | 3 | 3 | 3 | 3 | 1 | 0 |
12 | 3 | 1 | 1 | 3 | 3 | 2 | 0 |
13 | 1 | 2 | 1 | 2 | 1 | 1 | 0 |
14 | 3 | 2 | 2 | 2 | 1 | 1 | 0 |
15 | 2 | 2 | 1 | 1 | 2 | 2 | 0 |
16 | 3 | 1 | 1 | 3 | 3 | 1 | 0 |
17 | 1 | 1 | 2 | 2 | 2 | 1 | 0 |
青绿 1 | 蜷缩 1 | 浊响 1 | 清晰 1 | 凹陷 1 | 硬滑 1 | 好瓜 1 | |
乌黑 2 | 稍蜷 2 | 沉闷 2 | 稍糊 2 | 稍凹 2 | 软粘 2 | 坏瓜 2 | |
浅白 3 | 硬挺 3 | 清脆 3 | 模糊 3 | 平坦 3 |