分布问题 异常检测

#加载数据
import pandas as pd
import numpy as np
data = pd.read_csv('anomaly_data.csv')
data.head()

#数据可视化
%matplotlib inline
from matplotlib import pyplot as plt
fig1 = plt.figure(figsize=(10,7))
plt.scatter(data.loc[:,'x1'],data.loc[:,'x2'])
plt.title("data")
plt.xlabel('x1')
plt.ylabel('x2')
plt.show()

中心聚集的点是正常数据,旁边分散的点是异常数据

#define x1 and x2
x1 = data.loc[:,'x1']
x2 = data.loc[:,'x2']

fig2 = plt.figure(figsize=(10,5))
plt.subplot(121)
plt.hist(x1,bins=100)#100个数据分割
plt.title("x1 distribution")
plt.xlabel('x1')
plt.ylabel('counts')
plt.subplot(122)
plt.hist(x2,bins=100)#100个数据分割
plt.title("x2 distribution")
plt.xlabel('x2')
plt.ylabel('counts')
plt.show()

中心是最多的 再看高斯分布

#计算x1 x2 均值和标准差
x1_mean = x1.mean()
x1_sigma = x1.std()
x2_mean = x2.mean()
x2_sigma = x2.std()
print(x1_mean,x1_sigma,x2_mean,x2_sigma)

#计算高斯分布的概率密度 p(x)
from scipy.stats import norm
x1_range = np.linspace(0,20,300)#生成从0到20的300个连续的点 
x1_normal = norm.pdf(x1_range,x1_mean,x1_sigma)
x2_range = np.linspace(0,20,300)#生成从0到20的300个连续的点 
x2_normal = norm.pdf(x2_range,x2_mean,x2_sigma)

fig3 = plt.figure(figsize=(10,5))
plt.subplot(121)
plt.title("normal p(x1)")
plt.plot(x1_range,x1_normal)
plt.subplot(122)
plt.title("normal p(x2)")
plt.plot(x2_range,x2_normal)
plt.show()

训练

from sklearn.covariance import EllipticEnvelope
ad_model = EllipticEnvelope()
ad_model.fit(data)#无监督式学习

#预测
y_predict = ad_model.predict(data)
print(y_predict)
#1是正常点 -1是异常点
print(pd.value_counts(y_predict))

#画出异常数据的点
fig4 = plt.figure(figsize=(10,10))
original_data = plt.scatter(data.loc[:,'x1'],data.loc[:,'x2'],marker="x")
anomaly_data = plt.scatter(data.loc[:,'x1'][y_predict==-1],data.loc[:,'x2'][y_predict==-1],marker="o",facecolor='none',edgecolors='red',s=150)
plt.title("anomalu detection result")
plt.xlabel('x1')
plt.ylabel('x2')
plt.legend((original_data,anomaly_data),('original data','anomal data'))
plt.show()

修改阈值

ad_model = EllipticEnvelope(contamination=0.02)
ad_model.fit(data)

y_predict = ad_model.predict(data)

fig5= plt.figure(figsize=(10,10))
original_data = plt.scatter(data.loc[:,'x1'],data.loc[:,'x2'],marker="x")
anomaly_data = plt.scatter(data.loc[:,'x1'][y_predict==-1],data.loc[:,'x2'][y_predict==-1],marker="o",facecolor='none',edgecolors='red',s=150)
plt.title("anomalu detection result")
plt.xlabel('x1')
plt.ylabel('x2')
plt.legend((original_data,anomaly_data),('original data','anomal data'))
plt.show()