比赛的时间做法

企业排污时间序列异常检测

# import libraries
import matplotlib
import numpy as np
import pandas as pd
import seaborn as sns
from tensorflow.python.keras.layers import LSTM, RepeatVector, TimeDistributed
import matplotlib.pyplot as plt
import random

random.seed(12)
np.random.seed(12)
import tensorflow as tf

tf.random.set_seed(2021)
plt.style.use('seaborn')
sns.set(color_codes=True)
matplotlib.rcParams['font.sans-serif'] = ['SimHei']  # 用黑体显示中文

plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号
from sklearn.model_selection import train_test_split
from tensorflow.python.keras import Input, regularizers
from tensorflow.keras.layers import Dense
from tensorflow.python.keras.models import Model
from sklearn.preprocessing import StandardScaler

colums = ['公司名称', '排污口', '日期', '污染物名字', '排污浓度', '排污总量']
d = pd.read_csv('wushui.csv', names=colums)
# d = pd.read_csv('D:\项目\lstm异常检测\second1.csv', names=colums)
draw_plot = d.drop(['公司名称', '排污口', '污染物名字'], axis=1)
draw_plot = draw_plot.reset_index()
data = d.drop(['公司名称', '排污口', '污染物名字'], axis=1)
# data['month'] = data['日期'].map(lambda x: int(x[5:7]))
data = data.drop(['日期'], axis=1)
# data['month'] = StandardScaler().fit_transform(data[['month']])
# data['排污浓度'] = StandardScaler().fit_transform(data[['排污浓度']])
data['排污总量'] = StandardScaler().fit_transform(data[['排污总量']])
data_values = data.values
# 为无监督检测方法，并且按照8:2切成训练集和测试集
x_train, x_test = train_test_split(data, test_size=0.2, random_state=0)
X_train = x_train.values
test = x_test.values
data_values = data_values.reshape(data_values.shape[0], 1, data_values.shape[1])
X_train = X_train.reshape(X_train.shape[0], 1, X_train.shape[1])
print("Training data shape:", X_train.shape)
X_test = test.reshape(test.shape[0], 1, test.shape[1])
print("Test data shape:", X_test.shape)
# 检测异常的比例
percent = 0.1


def change_tf(x):
    if x:
        return 1
    else:
        return 0


def autoencoder_model(X):
    inputs = Input(shape=(X.shape[1], X.shape[2]))
    L1 = LSTM(100, activation='relu', return_sequences=True,
              kernel_regularizer=regularizers.l2(0.00))(inputs)
    L2 = LSTM(50, activation='relu', return_sequences=False)(L1)
    L3 = RepeatVector(X.shape[1])(L2)
    L4 = LSTM(50, activation='relu', return_sequences=True)(L3)
    L5 = LSTM(100, activation='relu', return_sequences=True)(L4)
    output = TimeDistributed(Dense(X.shape[2]))(L5)
    model = Model(inputs=inputs, outputs=output)
    return model


model = autoencoder_model(X_train)
model.compile(optimizer='adam', loss='mae')
model.summary()
history = model.fit(X_train, X_train, epochs=41, batch_size=32,
            validation_split=0.15).history
model.save('autoencoder_model.h5')
X_pred = model.predict(X_train)
print(X_pred)
X_pred = X_pred.reshape(X_pred.shape[0], X_pred.shape[2])
X_pred = pd.DataFrame(X_pred, columns=x_train.columns)
X_pred.index = x_train.index

scored = pd.DataFrame(index=x_train.index)
Xtrain = X_train.reshape(X_train.shape[0], X_train.shape[2])
scored['Loss_mae'] = np.mean(np.abs(X_pred - Xtrain), axis=1)

X_pred = model.predict(X_test)
X_pred = X_pred.reshape(X_pred.shape[0], X_pred.shape[2])
X_pred = pd.DataFrame(X_pred)
X_pred.index = x_test.index
scored = pd.DataFrame(index=x_test.index)
Xtest = X_test.reshape(X_test.shape[0], X_test.shape[2])
scored['Loss_mae'] = np.mean(np.abs(X_pred - Xtest), axis=1)
per = scored['Loss_mae'].quantile(percent)
scored['Threshold'] = per
scored['Anomaly'] = scored['Loss_mae'] > scored['Threshold']

X_pred = model.predict(X_train)
X_pred = X_pred.reshape(X_pred.shape[0], X_pred.shape[2])
X_pred = pd.DataFrame(X_pred, columns=x_train.columns)
X_pred.index = x_train.index
scored_train = pd.DataFrame(index=x_train.index)
scored_train['Loss_mae'] = np.mean(np.abs(X_pred - Xtrain), axis=1)
per = scored_train['Loss_mae'].quantile(percent)
scored_train['Threshold'] = per
scored_train['Anomaly'] = scored_train['Loss_mae'] > scored_train['Threshold']
scored = pd.concat([scored_train, scored])

all_pred = model.predict(data_values)
all_pred = all_pred.reshape(all_pred.shape[0], all_pred.shape[2])
all_pred = pd.DataFrame(all_pred, columns=data.columns)
all_pred.index = data.index
scored_all = pd.DataFrame(index=data.index)
scored_all = scored_all.reset_index()
all_data = data_values.reshape(data_values.shape[0], data_values.shape[2])
scored_all['Loss_mae'] = np.mean(np.abs(all_pred - all_data), axis=1)
per = scored_all['Loss_mae'].quantile(percent)
scored_all['Threshold'] = per
scored_all['Anomaly'] = scored_all['Loss_mae'] > scored_all['Threshold']
scored_all['Anomaly'] = scored_all['Anomaly'].map(lambda x: change_tf(x))
all_data_ = pd.merge(draw_plot, scored_all, left_on='index', right_on='index')
Anomaly_data = all_data_[all_data_['Anomaly'] == 0]
normal_data = all_data_[all_data_['Anomaly'] == 1]
Anomaly_data.to_csv('Anomaly_data.csv', index=False)
normal_data.to_csv('normal_data.csv', index=False)
inlier = normal_data.values
outliers = Anomaly_data.values
title = '异常检测'
plt.title(title)
b1 = plt.scatter(inlier[:, 0], inlier[:, 3], c='green',
                 s=20, edgecolor='k')
c = plt.scatter(outliers[:, 0], outliers[:, 3], c='red',
                s=20, edgecolor='k')
plt.axis('tight')
plt.xlabel('index')
plt.ylabel('排污总量')
plt.legend([b1, c],
           ["normal", "abnormal"],
           loc="upper left")

plt.show()