Network Traffic Anomaly Detection Using Recurrent Neural Networks
import numpy as np
import pandas as pd
import re
import h5py
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.metrics import log_loss, auc, roc_curve
from tensorflow.python.keras.layers.core import Masking
from tensorflow.python.keras.layers import Dense, LSTM, Dropout, Embedding
from tensorflow.python.keras.layers.wrappers import Bidirectional
from tensorflow.python.keras.models import Model, Sequential
from tensorflow.python.keras.utils import np_utils
from tensorflow.python.keras.models import Model, Sequential
from tensorflow.python.keras.layers import Embedding, TimeDistributed
from tensorflow.python.keras.models import load_model
from tensorflow.python.client import device_lib
from lxml import etree
from itertools import groupby
from gensim.models import Word2Vec
import glob
import math
import itertools
from sklearn.metrics import *
import matplotlib.pyplot as plt
##
## Read in the raw ISCX IDS Data
##
print("Loading data...")
data = pd.read_csv('data4.csv')
# xml_list = glob.glob('data/labeled_flows_xml/*xml')
# parser = etree.XMLParser(recover=True)
# def xml2df(xml_data):
# root = etree.fromstring(xml_data, parser=parser) # element tree
# all_records = []
# for i, child in enumerate(root):
# record = {}
# for subchild in child:
# record[subchild.tag] = subchild.text
# all_records.append(record)
# return pandas.DataFrame(all_records)
# dfs = []
# for ii in xml_list:iteritems
# xml_data = open(ii).read()
# dfs.append(xml2df(xml_data))
# data = pandas.concat(dfs)
# data = data.drop_duplicates()
# del dfs
##
## Produce undirected IP-dyads and order by time
##
print("De-dup Flows: "+str(len(data)))
print("Creating undirected IP-dyads...")
data = data.sort_values('startDateTime')
data['seqId'] = data['source'] + '_' + data['destination'] + '_' + data['startDateTime'].str[:13]
data['lowPort'] = np.where(data.destinationPort <= data.sourcePort, data['destinationPort'], data['sourcePort'])
##
## Build hour-IP-dyad keys and sequences
##
print("Building sequences...")
key = data.groupby('seqId')[['Tag','lowPort']].agg({"Tag":lambda x: "%s" % ','.join([a for a in x]),"lowPort":lambda x: "%s" % ','.join([str(a) if int(a)<10000 else "10000" for a in x])})
print("Unique Keys: "+str(key.count()))
attacks = [a.split(",") for a in key.Tag.tolist()]
sequences = [a.split(",") for a in key.lowPort.tolist()]
##
## Create Label Encoder and add one to account for 0. masking
##
print("Generating Label Encoder...")
unique_tokens = list(set([a for b in sequences for a in b]))
le = LabelEncoder()
le.fit(unique_tokens)
sequences = [le.transform(s).tolist() for s in sequences]
sequences = [[b for b in a] for a in sequences]
sequence_attack = zip(attacks, sequences)
##
## Build sequences
##
print("Generating sequences for model...")
na_value = 0.
seq_len = 10
seq_index = []
seq_x = []
seq_y = []
seq_attack = []
for si, (sa, ss) in enumerate(sequence_attack):
prepend = [0.] * (seq_len)
seq = prepend + ss
seqa = prepend + sa
for ii in range(seq_len, len(seq)):
subseq = seq[(ii-seq_len):(ii)]
vex = []
for ee in subseq:
try:
vex.append(ee)
except:
vex.append(na_value)
seq_x.append(vex)
seq_y.append(seq[ii])
seq_index.append(si)
seq_attack.append(seqa[ii])
##
## Make One-hot-encoder
##
print("Initializing One-hot-encoder...")
ohe = OneHotEncoder(sparse=False)
ohe.fit(np.unique(seq_y).reshape(-1,1))
X = np.array(seq_x)
##
## Generator for Batch Training
##
class BatchGenerator(object):
def __init__(self, batch_size, x, y, ohe):
self.batch_size = batch_size
self.n_batches = int(math.floor(np.shape(x)[0] / batch_size))
self.batch_index = [a * batch_size for a in range(0, self.n_batches)]
self.x = x
self.y = y
self.ohe = ohe
def __iter__(self):
for bb in itertools.cycle(self.batch_index):
y = self.y[bb:(bb+self.batch_size)]
ohe_y = self.ohe.transform(y.reshape(len(y), 1))
yield (self.x[bb:(bb+self.batch_size),], ohe_y)/usr/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject
return f(*args, **kwds)
/usr/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject
return f(*args, **kwds)
/usr/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject
return f(*args, **kwds)
Loading data...
De-dup Flows: 196023
Creating undirected IP-dyads...
Building sequences...
Unique Keys: Tag 9075
lowPort 9075
dtype: int64
Generating Label Encoder...
Generating sequences for model...
Initializing One-hot-encoder...
##
## Define model
##
print("Defining model...")
model = Sequential()
model.add(Embedding(output_dim=100, input_dim=len(unique_tokens), mask_zero=True))
model.add(Bidirectional(LSTM(50, return_sequences=True)))
model.add(Dropout(0.2))
model.add(Bidirectional(LSTM(50, activation="relu", return_sequences=False)))
model.add(Dropout(0.2))
model.add(Dense(50, activation="linear"))
model.add(Dropout(0.2))
model.add(Dense(len(unique_tokens), activation="softmax"))
model.compile(loss="categorical_crossentropy", optimizer="nadam", metrics=["accuracy"])
training_data = BatchGenerator(512, np.asarray(X), np.asarray(seq_y), ohe)
model.fit_generator(training_data.__iter__(),
steps_per_epoch=training_data.n_batches,
epochs=1, verbose=1)Defining model...
WARNING:tensorflow:From <ipython-input-2-35991dfbf642>:21: Model.fit_generator (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.
Instructions for updating:
Please use Model.fit, which supports generators.
382/382 [==============================] - 60s 157ms/step - loss: 0.8326 - accuracy: 0.9346
Out[2]:
<tensorflow.python.keras.callbacks.History at 0x7f86a836d518>model.save("models/ports_dirty.hd5")
#model = load_model("models/ports_dirty.hd5")
preds = model.predict_proba(X, batch_size=32)
indexed_preds = zip(np.asarray(seq_index), preds, np.asarray(seq_y), np.asarray(seq_attack))WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/tracking/tracking.py:111: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.
Instructions for updating:
This property should not be used in TensorFlow 2.0, as updates are applied automatically.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/tracking/tracking.py:111: Layer.updates (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.
Instructions for updating:
This property should not be used in TensorFlow 2.0, as updates are applied automatically.
INFO:tensorflow:Assets written to: models/ports_dirty.hd5/assets
WARNING:tensorflow:From <ipython-input-3-63b07c64bf5d>:3: Sequential.predict_proba (from tensorflow.python.keras.engine.sequential) is deprecated and will be removed after 2021-01-01.
Instructions for updating:
Please use `model.predict()` instead.print(type(seq_index))
print(type(preds))
print(type(seq_y))
print(type(seq_attack))
print(np.asarray(seq_index).shape,np.asarray(preds).shape,np.asarray(seq_y).shape,np.asarray(seq_attack).shape)
print(np.asarray(seq_index)[0:3],np.asarray(preds[0:3]),np.asarray(seq_y)[0:3],np.asarray(seq_attack)[0:3])
print(pd.value_counts(np.asarray(seq_attack)))<class 'list'>
<class 'numpy.ndarray'>
<class 'list'>
<class 'list'>
(196023,) (196023, 426) (196023,) (196023,)
[0 0 0] [[0.00300472 0.00425819 0.00202167 ... 0.00200748 0.00221202 0.0023317 ]
[0.00380625 0.00765104 0.00138027 ... 0.00131454 0.00165566 0.00209766]
[0.00334394 0.00842383 0.0006377 ... 0.00058746 0.00080875 0.00124958]] [406 406 406] ['Normal' 'Normal' 'Normal']
Normal 158565
Attack 37458
dtype: int64logloss_list = []
for (ii, pp, yy, aa) in indexed_preds:
ll = -math.log(pp[yy-1]+1e-10)
logloss_list.append(ll)
fpr, tpr, thresholds = roc_curve(np.asarray(seq_attack),logloss_list, pos_label="Attack")
plt.figure()
lw = 2
plt.plot(fpr, tpr, color='darkorange',
lw=lw, label='ROC curve (area = %0.2f)' % auc(fpr,tpr))
plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
plt.legend(loc="lower right")
plt.show()
logloss_list = seq_index
fpr, tpr, thresholds = roc_curve(np.asarray(seq_attack),logloss_list, pos_label="Attack") # 根据AUC曲线可以间接看出两个变量之间是否有关系。标签数据和自变量组成的列表
plt.figure()
lw = 2
plt.plot(fpr, tpr, color='darkorange',
lw=lw, label='ROC curve (area = %0.2f)' % auc(fpr,tpr))
plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
plt.legend(loc="lower right")
plt.show()
print(type(fpr))
print(fpr.shape,tpr.shape)
print(fpr)
print(type(logloss_list))
print(len(logloss_list))
print(logloss_list[0:10])<class 'numpy.ndarray'>
(192,) (192,)
[0. 0.12400266 0.12444592 0.12455674 0.12488918 0.12544326
0.12566489 0.12632979 0.13741135 0.13973848 0.15403369 0.1548094
0.15503103 0.15569592 0.15591755 0.15724734 0.17508865 0.17675089
0.17697252 0.17763741 0.20700355 0.20789007 0.23404255 0.23459663
0.29787234 0.29964539 0.29997784 0.30086436 0.30108599 0.30141844
0.30164007 0.30208333 0.36724291 0.36735372 0.36757535 0.3685727
0.36901596 0.3693484 0.36957004 0.37012411 0.37034574 0.37067819
0.37089982 0.37101064 0.37211879 0.37234043 0.37267287 0.37278369
0.37300532 0.37322695 0.3785461 0.37887855 0.37921099 0.37976507
0.38087323 0.38120567 0.38264628 0.38286791 0.38308954 0.38475177
0.3849734 0.38541667 0.38552748 0.38619238 0.3870789 0.38718972
0.38796543 0.38818706 0.60449911 0.60505319 0.60527482 0.60859929
0.60959663 0.61136968 0.61170213 0.61192376 0.61281028 0.61314273
0.61325355 0.61347518 0.61447252 0.61458333 0.61480496 0.61591312
0.61735372 0.61835106 0.6185727 0.6193484 0.62156472 0.62178635
0.62189716 0.62555408 0.62588652 0.62610816 0.62666223 0.62754876
0.62799202 0.62810284 0.6299867 0.63042996 0.63087323 0.63109486
0.63242465 0.63253546 0.63253546 0.63552748 0.63696809 0.6391844
0.63940603 0.63951684 0.64095745 0.64350621 0.64383865 0.64406028
0.64461436 0.64472518 0.64494681 0.64527926 0.64550089 0.64572252
0.64882535 0.64893617 0.64949025 0.65082004 0.6512633 0.65381206
0.65702571 0.65713652 0.65824468 0.66156915 0.66256649 0.6626773
0.6626773 0.66888298 0.66910461 0.66910461 0.67309397 0.68517287
0.68517287 0.6853945 0.6853945 0.69636525 0.69636525 0.6995789
0.6995789 0.69968972 0.69968972 0.92741578 0.92741578 0.9275266
0.9275266 0.92796986 0.92907801 0.92918883 0.92929965 0.92929965
0.92963209 0.92963209 0.92974291 0.93218085 0.93229167 0.93229167
0.93262411 0.93262411 0.93295656 0.93295656 0.93306738 0.93306738
0.93339982 0.93339982 0.9339539 0.9339539 0.93406472 0.95057624
0.95057624 0.95079787 0.9510195 0.9510195 0.95113032 0.95467642
0.97163121 0.97174202 0.97174202 0.97218528 0.98071809 0.98071809
0.98304521 0.98315603 0.984375 0.99911348 0.99977837 1. ]
<class 'list'>
196023
[6.119278511493977, 6.330971669458178, 6.958946746200708, 7.783773825936498, 8.415765024293194, 8.80955451083171, 9.041169291392345, 9.171438551724645, 6.119278511493977, 6.330971669458178]key_ll = zip(seq_index, logloss_list, seq_attack)
dictionary = dict()
for (key, ll, aa) in key_ll: #
current_value = dictionary.get(str(key), ([],[]))
dictionary[str(key)] = (current_value[0] + [ll], current_value[1] + [aa])
agg_ll = []
agg_bad = []
for key, val in dictionary.items():
bad = str(np.mean([v=="Attack" for v in val[1]]) > 0.)
score = np.max(val[0])
agg_bad.append(bad)
agg_ll.append(score)
fpr, tpr, thresholds = roc_curve(agg_bad, agg_ll, pos_label="True")
plt.figure()
lw = 2
plt.plot(fpr, tpr, color='darkorange',
lw=lw, label='ROC curve (area = %0.2f)' % auc(fpr,tpr))
plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Ports Dirty Baseline')
plt.legend(loc="lower right")
plt.savefig("graphics/ports_dirty_ipdyadhour-wise.pdf", format="pdf")
plt.show()
print(np.asarray(seq_index).shape,np.asarray(logloss_list).shape,np.asarray(seq_attack).shape)
print(pd.value_counts(np.asarray(seq_index)))(196023,) (196023,) (196023,)
5213 9889
3970 7725
3481 6039
6430 4843
7749 4596
...
5277 1
7324 1
1055 1
6893 1
5949 1
Length: 9075, dtype: int64
In [18]:
print(seq_index)[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 24, 25, 26, 27, 27, 27, 27, 27, 27, 28, 28, 28, 28, 28, 28, 29, 30, 30, 31, 31, 32, 33, 34, 35, 35, 35, 35, 35, 36, 36, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38,...
key_ll = zip(seq_index, logloss_list, seq_attack)
dictionary = dict()
for (key, ll, aa) in key_ll: #
current_value = dictionary.get(str(key), ([],[]))
dictionary[str(key)] = (current_value[0] + [ll], current_value[1] + [aa])
for key,value in dictionary.items():
print(key,value)0 ([0, 0, 0, 0, 0, 0, 0, 0], ['Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal']) 1 ([1, 1, 1, 1, 1, 1, 1, 1, 1, 1], ['Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal']) 2 ([2, 2, 2, 2, 2, 2, 2, 2], ['Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal']) 3 ([3, 3, 3, 3, 3, 3, 3, 3], ['Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal']) 4 ([4, 4, 4, 4, 4, 4, 4, 4, 4, 4], ['Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal']) 5 ([5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], ['Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal']) 6 ([6, 6, 6, 6, 6, 6, 6, 6, 6, 6], ['Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal']) 7 ([7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7], ['Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal']) 8 ([8, 8, 8, 8, 8, 8, 8, 8, 8, 8], ['Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal', 'Normal']) 9 ([9, 9, 9, 9, 9], ['Normal', 'Normal', 'Normal', 'Normal', 'Normal']) 10 ([10, 10, 10, 10], ['Normal', 'Normal', 'Normal', 'Normal']) 11 ([11, 11], ['Normal', 'Normal']) 12 ([12, 12, 12, 12], ['Normal', 'Normal', 'Normal', 'Normal']) 13 ([13, 13, 13, 13], ['Normal', 'Normal', 'Normal', 'Normal']) 14 ([14, 14, 14, 14], ['Normal', 'Normal', 'Normal', 'Normal'])...
826
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
