#Modified and re-adapted by fullarray

from keras.utils.data_utils import get_file

try:

path = get_file('kddcup.data_10_percent.gz', origin='http://kdd.ics.uci.edu/databases/kddcup99/kddcup.data_10_percent.gz')

except:

print('Error downloading')

raise

print(path)

df = pd.read_csv(path, header=None)

print("Read {} rows. ". format(len(df)))

df.dropna(inplace=True, axis=1)

#add headers as there are any in the dataset.

df.columns = [

'duration',

'protocol_type',

'service',

'flag',

'src_bytes',

'dst_bytes',

'land',

'wrong_fragment',

'urgent',

'hot',

'num_failed_logins',

'logged_in',

'num_compromised',

'root_shell',

'su_attempted',

'num_root',

'num_file_creations',

'num_shells',

'num_access_files',

'num_outbound_cmds',

'is_host_login',

'is_guest_login',

'count',

'srv_count',

'serror_rate',

'srv_serror_rate',

'rerror_rate',

'srv_rerror_rate',

'same_srv_rate',

'diff_srv_rate',

'srv_diff_host_rate',

'dst_host_count',

'dst_host_srv_count',

'dst_host_same_srv_rate',

'dst_host_diff_srv_rate',

'dst_host_same_src_port_rate',

'dst_host_srv_diff_host_rate',

'dst_host_serror_rate',

'dst_host_srv_serror_rate',

'dst_host_rerror_rate',

'dst_host_srv_rerror_rate',

'outcome'

]

#display 5 rows

df[0:5]

#start analysis

ENCODING = 'utf-8'

def expand_categories(values):

result = []

s = values.value_counts()

t = float(len(values))

for v in s.index:

result.append("{}:{}%".format(v, round(100*(s[v]/t),2)))

return "[{}]".format(",".join(result))

def analyze(filename):

print()

print("Analyzing: {}".format(filename))

df = pd.read_csv(filename,encoding=ENCODING)

cols = df.columns.values

total = float(len(df))

print("{} rows".format(int(total)))

for col in cols:

uniques = df[col].unique()

unique_count = len(uniques)

if unique_count>100:

print("** {}:{} ({}%)".format(col, unique_count, int(((unique_count)/total)*100)))

else:

print("** {}:{}".format(col,expand_categories(df[col])))

expand_categories(df[col])

#Analyze actual kdd dataset(partial amount of records)

import tensorflow.contrib.learn as skflow

import pandas as pd

import os

import numpy as np

from sklearn import metrics

from scipy.stats import zscore

path = "./data/"

filename_read = os.path.join(path,"auto-mpg.csv")

# Now encode the feature vector

encode_numeric_zscore(df, 'duration')

encode_text_dummy(df, 'protocol_type')

encode_text_dummy(df, 'service')

encode_text_dummy(df, 'flag')

encode_numeric_zscore(df, 'src_bytes')

encode_numeric_zscore(df, 'dst_bytes')

encode_text_dummy(df, 'land')

encode_numeric_zscore(df, 'wrong_fragment')

encode_numeric_zscore(df, 'urgent')

encode_numeric_zscore(df, 'hot')

encode_numeric_zscore(df, 'num_failed_logins')

encode_text_dummy(df, 'logged_in')

encode_numeric_zscore(df, 'num_compromised')

encode_numeric_zscore(df, 'root_shell')

encode_numeric_zscore(df, 'su_attempted')

encode_numeric_zscore(df, 'num_root')

encode_numeric_zscore(df, 'num_file_creations')

encode_numeric_zscore(df, 'num_shells')

encode_numeric_zscore(df, 'num_access_files')

encode_numeric_zscore(df, 'num_outbound_cmds')

encode_text_dummy(df, 'is_host_login')

encode_text_dummy(df, 'is_guest_login')

encode_numeric_zscore(df, 'count')

encode_numeric_zscore(df, 'srv_count')

encode_numeric_zscore(df, 'serror_rate')

encode_numeric_zscore(df, 'srv_serror_rate')

encode_numeric_zscore(df, 'rerror_rate')

encode_numeric_zscore(df, 'srv_rerror_rate')

encode_numeric_zscore(df, 'same_srv_rate')

encode_numeric_zscore(df, 'diff_srv_rate')

encode_numeric_zscore(df, 'srv_diff_host_rate')

encode_numeric_zscore(df, 'dst_host_count')

encode_numeric_zscore(df, 'dst_host_srv_count')

encode_numeric_zscore(df, 'dst_host_same_srv_rate')

encode_numeric_zscore(df, 'dst_host_diff_srv_rate')

encode_numeric_zscore(df, 'dst_host_same_src_port_rate')

encode_numeric_zscore(df, 'dst_host_srv_diff_host_rate')

encode_numeric_zscore(df, 'dst_host_serror_rate')

encode_numeric_zscore(df, 'dst_host_srv_serror_rate')

encode_numeric_zscore(df, 'dst_host_rerror_rate')

encode_numeric_zscore(df, 'dst_host_srv_rerror_rate')

outcomes = encode_text_index(df, 'outcome')

num_classes = len(outcomes)

#Show 5 rows

df.dropna(inplace=True, axis=1)

df[0:5]

#Start training the neural network

import pandas as pd

import io

import requests

import numpy as np

import os

from sklearn.model_selection import train_test_split

from sklearn import metrics

from keras.models import Sequential

from keras.layers.core import Dense, Activation

from keras.callbacks import EarlyStopping

#Break into X (predictiors) & y (prediction)

x, y = to_xy(df, 'outcome')

#Create a test/train split. 25% test

#Split the dataset into a train and test subsets

x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25, random_state=42)

#Make the actual neural network

#Model begins here

model = Sequential()

model.add(Dense(10, input_dim=x.shape[1], kernel_initializer='normal', Activation='relu'))

model.add(Dense(50, input_dim=x.shape[1], kernel_initialize='normal', Activation='relu'))

model.add(Dense(10, input_dim=x.shape[1], kernel_initialize='normal', Activation='relu'))

model.add(Dense(1, kernel_initializer='normal'))

model.add(Dense(y.shape[1], Activation='softmax'))

model.compile(loss='categorical_crossentropy', optimizer='adam')

monitor = EarlyStopping(monitor='val_loss', min_delta=le-3, patience=5, verbose=1, mode='auto')

model.fit(x_train, y_train, validation_data=(x_test, y_test), callbacks=[monitor], verbose=2, epochs=1000)

#Model ends here

#Check accuracy of the learning by predicting first

pred = model.predict(x_test)

pred = np.argmax(pred,axis=1)

y_eval = np.argmax(y_test, axis=1)

score = metrics.accuracy_score(y_eval, pred)

#Show final results

print("Validation score: {}".format(score))