流程
- 数据清洗(Data Cleaning)& 特征工程(Feature Engineering)
- 基本建模&评估(Basic Modeling & Evaluation)
- 预测(Predict)
- 确定节点,确定以哪些特征(属性)为节点。
- 确定结构,确定网络结构(拓扑)用以反应变量节点之间的依赖关系。
- 确定参数,明确每条边上的条件概率。
节点选择: 这里还包括由特征工程特征选择之类的工作。当然若有专业知识的参与会得到更合理的特征选择。
网络结构&参数:a. 专家设计
b. 数据生成
数据清理
# dataset/titanic/train.csv
# dataset/titanic/test.csv
import pandas as pd
TRAIN = 'dataset/titanic/train.csv'
TEST = 'dataset/titanic/test.csv'
'''
PassengerId => 乘客ID
Pclass => 客舱等级(1/2/3等舱位)
Name => 乘客姓名
Sex => 性别 清洗成male=1 female=0
Age => 年龄 插补后分0,1,2 代表 幼年(0-15) 成年(15-55) 老年(55-)
SibSp => 兄弟姐妹数/配偶数
Parch => 父母数/子女数
Ticket => 船票编号
Fare => 船票价格 经聚类变0 1 2 代表少 多 很多
Cabin => 客舱号 清洗成有无此项,并发现有的生存率高
Embarked => 登船港口 清洗na,填S
'''
def girl(aa):
if (aa.Age != 999) & (aa.Title == 'Miss') & (aa.Age <= 14):
return 'Girl'
elif (aa.Age == 999) & (aa.Title == 'Miss') & (aa.Parch != 0):
return 'Girl'
else:
return aa.Title
def clean():
# combine train and test set.
train = pd.read_csv(TRAIN)
test = pd.read_csv(TEST)
full = pd.concat([train, test], ignore_index=True)
full['Embarked'].fillna('S', inplace=True)
full.Fare.fillna(full[full.Pclass == 3]['Fare'].median(), inplace=True)
full.loc[full.Cabin.notnull(), 'Cabin'] = 1
full.loc[full.Cabin.isnull(), 'Cabin'] = 0
full.loc[full['Sex'] == 'male', 'Sex'] = 1
full.loc[full['Sex'] == 'female', 'Sex'] = 0
full['Title'] = full['Name'].apply(lambda x: x.split(',')[1].split('.')[0].strip())
nn = {'Capt': 'Rareman', 'Col': 'Rareman', 'Don': 'Rareman', 'Dona': 'Rarewoman',
'Dr': 'Rareman', 'Jonkheer': 'Rareman', 'Lady': 'Rarewoman', 'Major': 'Rareman',
'Master': 'Master', 'Miss': 'Miss', 'Mlle': 'Rarewoman', 'Mme': 'Rarewoman',
'Mr': 'Mr', 'Mrs': 'Mrs', 'Ms': 'Rarewoman', 'Rev': 'Mr', 'Sir': 'Rareman',
'the Countess': 'Rarewoman'}
full.Title = full.Title.map(nn)
# assign the female 'Dr' to 'Rarewoman'
full.loc[full.PassengerId == 797, 'Title'] = 'Rarewoman'
full.Age.fillna(999, inplace=True)
full['Title'] = full.apply(girl, axis=1)
Tit = ['Mr', 'Miss', 'Mrs', 'Master', 'Girl', 'Rareman', 'Rarewoman']
for i in Tit:
full.loc[(full.Age == 999) & (full.Title == i), 'Age'] = full.loc[full.Title == i, 'Age'].median()
full.loc[full['Age'] <= 15, 'Age'] = 0
full.loc[(full['Age'] > 15) & (full['Age'] < 55), 'Age'] = 1
full.loc[full['Age'] >= 55, 'Age'] = 2
full['Pclass'] = full['Pclass'] - 1
from sklearn.cluster import KMeans
Fare = full['Fare'].values
Fare = Fare.reshape(-1, 1)
km = KMeans(n_clusters=3).fit(Fare) # 将数据集分为2类
Fare = km.fit_predict(Fare)
full['Fare'] = Fare
full['Fare'] = full['Fare'].astype(int)
full['Age'] = full['Age'].astype(int)
full['Cabin'] = full['Cabin'].astype(int)
full['Pclass'] = full['Pclass'].astype(int)
full['Sex'] = full['Sex'].astype(int)
# full['Survived']=full['Survived'].astype(int)
dataset = full.drop(columns=['Embarked', 'Name', 'Parch', 'PassengerId', 'SibSp', 'Ticket', 'Title'])
dataset.dropna(inplace=True)
dataset['Survived'] = dataset['Survived'].astype(int)
# dataset=pd.concat([dataset, pd.DataFrame(columns=['Pri'])])
train = dataset[:800]
test = dataset[800:]
return train,test
'''
最后保留如下项目,并切出800的训练集:
Pclass => 客舱等级(0/1/2等舱位)
Sex => 性别 male=1 female=0
Age => 年龄 插补后分0,1,2 代表 幼年(0-15) 成年(15-55) 老年(55-)
Fare => 船票价格 经聚类变0 1 2 代表少 多 很多
Cabin => 客舱号 清洗成有无此项,并发现有的生存率高
'''
结构搭建,参数学习
from pgmpy.models import BayesianModel
from pgmpy.estimators import BayesianEstimator
# 模型建立
def buildStructure():
#model = BayesianModel([('Age', 'Pri'), ('Sex', 'Pri'),('Pri','Survived'),('Fare','Pclass'),('Pclass','Survived'),('Cabin','Survived')])
model = BayesianModel([('Age', 'Survived'),
('Sex', 'Survived'),
('Fare', 'Pclass'),
('Pclass', 'Survived'),
('Cabin', 'Survived')])
# (‘Age’, ‘Survived’)表示Age指向Survived
return model
# 参数学习
def learnParmater(model,train):
# default equivalent_sample_size=5
model.fit(train, estimator=BayesianEstimator, prior_type="BDeu")
# 预测函数
def predict(model, test):
predict_data=test.drop(columns=['Survived'],axis=1)
y_pred = model.predict(predict_data)
# print(y_pred)
print(type(y_pred['Survived'].tolist()),type(test['Survived']))
out = y_pred['Survived'].tolist()
tes = test['Survived'].tolist()
count =0
for ind in range(0,len(y_pred['Survived'].tolist())):
if out[ind] == tes[ind]:
count = count + 1
print(count/len(test))
调用
from dataClean import clean
from structure import buildStructure,learnParmater,predict
# 数据清理,特征选择
train, test = clean()
# 结构建立
model = buildStructure()
# 参数学习
learnParmater(model,train)
#输出节点信息
print(model.nodes())
#输出依赖关系
print(model.edges())
#查看某节点概率分布
print(model.get_cpds('Pclass').values)
# 预测
predict(model, test)
问题
使用pgmpy报错 cannot import name ‘BDeuScore’ from ‘pgmpy.estimators.BDeuScore’
解决方案