ids_mal.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "advance-interest",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Extraction connections not labeled BENIGN from the CICIDS 2017 dataset\n",
    "Treating each malicious connection type as a separate class, there are 14 classes: 1 - 14\n",
    "The input and output are in local directories\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ranking-termination",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the top modules that are used in multiple places\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "precious-comparison",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Column name mapping from original data to compact form\n",
    "# All the X** are features and the YY is the label\n",
    "feature_map = {\n",
    " ' Destination Port' : 'X1',\n",
    " ' Flow Duration' : 'X2', \n",
    " ' Total Fwd Packets' : 'X3', \n",
    " ' Total Backward Packets' : 'X4', \n",
    " 'Total Length of Fwd Packets' : 'X5', \n",
    " ' Total Length of Bwd Packets' : 'X6', \n",
    " ' Fwd Packet Length Max' : 'X7', \n",
    " ' Fwd Packet Length Min' : 'X8', \n",
    " ' Fwd Packet Length Mean' : 'X9', \n",
    " ' Fwd Packet Length Std' : 'X10', \n",
    " 'Bwd Packet Length Max' : 'X11', \n",
    " ' Bwd Packet Length Min' : 'X12', \n",
    " ' Bwd Packet Length Mean' : 'X13', \n",
    " ' Bwd Packet Length Std' : 'X14', \n",
    " 'Flow Bytes/s' : 'X15', \n",
    " ' Flow Packets/s' : 'X16', \n",
    " ' Flow IAT Mean' : 'X17', \n",
    " ' Flow IAT Std' : 'X18', \n",
    " ' Flow IAT Max' : 'X19', \n",
    " ' Flow IAT Min' : 'X20', \n",
    " 'Fwd IAT Total' : 'X21', \n",
    " ' Fwd IAT Mean' : 'X22', \n",
    " ' Fwd IAT Std' : 'X23', \n",
    " ' Fwd IAT Max' : 'X24', \n",
    " ' Fwd IAT Min' : 'X25', \n",
    " 'Bwd IAT Total' : 'X26', \n",
    " ' Bwd IAT Mean' : 'X27', \n",
    " ' Bwd IAT Std' : 'X28', \n",
    " ' Bwd IAT Max' : 'X29', \n",
    " ' Bwd IAT Min' : 'X30', \n",
    " 'Fwd PSH Flags' : 'X31', \n",
    " ' Bwd PSH Flags' : 'X32', \n",
    " ' Fwd URG Flags' : 'X33', \n",
    " ' Bwd URG Flags' : 'X34', \n",
    " ' Fwd Header Length' : 'X35', \n",
    " ' Bwd Header Length' : 'X36', \n",
    " 'Fwd Packets/s' : 'X37', \n",
    " ' Bwd Packets/s' : 'X38', \n",
    " ' Min Packet Length' : 'X39', \n",
    " ' Max Packet Length' : 'X40', \n",
    " ' Packet Length Mean' : 'X41', \n",
    " ' Packet Length Std' : 'X42', \n",
    " ' Packet Length Variance' : 'X43', \n",
    " 'FIN Flag Count' : 'X44', \n",
    " ' SYN Flag Count' : 'X45', \n",
    " ' RST Flag Count' : 'X46', \n",
    " ' PSH Flag Count' : 'X47', \n",
    " ' ACK Flag Count' : 'X48', \n",
    " ' URG Flag Count' : 'X49', \n",
    " ' CWE Flag Count' : 'X50', \n",
    " ' ECE Flag Count' : 'X51', \n",
    " ' Down/Up Ratio' : 'X52', \n",
    " ' Average Packet Size' : 'X53', \n",
    " ' Avg Fwd Segment Size' : 'X54', \n",
    " ' Avg Bwd Segment Size' : 'X55', \n",
    " ' Fwd Header Length.1' : 'X56', \n",
    " 'Fwd Avg Bytes/Bulk' : 'X57', \n",
    " ' Fwd Avg Packets/Bulk' : 'X58', \n",
    " ' Fwd Avg Bulk Rate' : 'X59', \n",
    " ' Bwd Avg Bytes/Bulk' : 'X60', \n",
    " ' Bwd Avg Packets/Bulk' : 'X61', \n",
    " 'Bwd Avg Bulk Rate' : 'X62', \n",
    " 'Subflow Fwd Packets' : 'X63', \n",
    " ' Subflow Fwd Bytes' : 'X64', \n",
    " ' Subflow Bwd Packets' : 'X65', \n",
    " ' Subflow Bwd Bytes' : 'X66', \n",
    " 'Init_Win_bytes_forward' : 'X67', \n",
    " ' Init_Win_bytes_backward' : 'X68', \n",
    " ' act_data_pkt_fwd' : 'X69', \n",
    " ' min_seg_size_forward' : 'X70', \n",
    " 'Active Mean' : 'X71', \n",
    " ' Active Std' : 'X72', \n",
    " ' Active Max' : 'X73', \n",
    " ' Active Min' : 'X74', \n",
    " 'Idle Mean' : 'X75', \n",
    " ' Idle Std' : 'X76', \n",
    " ' Idle Max' : 'X77', \n",
    " ' Idle Min' : 'X78', \n",
    " ' Label': 'YY'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "serial-hudson",
   "metadata": {},
   "outputs": [],
   "source": [
    "# label names (YY) in the data and their\n",
    "# mapping to numerical values\n",
    "label_map = {\n",
    " 'BENIGN' : 1,\n",
    " 'FTP-Patator' : 2,\n",
    " 'SSH-Patator' : 3,\n",
    " 'DoS slowloris' : 4,\n",
    " 'DoS Slowhttptest': 5,\n",
    " 'DoS Hulk' : 6,\n",
    " 'DoS GoldenEye' : 7,\n",
    " 'Heartbleed' : 8,\n",
    " 'Web Attack � Brute Force' : 9,\n",
    " 'Web Attack � XSS' : 10,\n",
    " 'Web Attack � Sql Injection' : 11,\n",
    " 'Infiltration' : 12,\n",
    " 'Bot' : 13,\n",
    " 'PortScan' : 14,\n",
    " 'DDoS' : 15,\n",
    "}\n",
    "\n",
    "num_ids_features = 78\n",
    "num_ids_classes = 15\n",
    "ids_classes = [ 'BENIGN', 'FTP-Patator', 'SSH-Patator', 'DoS slowloris', 'DoS Slowhttptest', 'DoS Hulk', 'DoS GoldenEye', 'Heartbleed', 'Brute Force', 'XSS', 'Sql Injection', 'Infiltration', 'Bot', 'PortScan', 'DDoS',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "pointed-basin",
   "metadata": {},
   "outputs": [],
   "source": [
    "indir = './MachineLearningCVE/raw/'\n",
    "outdir = './MachineLearningCVE/restart/'\n",
    "mal_data = 'mal.csv'\n",
    "in_filenames = [\n",
    "    'Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv', \n",
    "    'Monday-WorkingHours.pcap_ISCX.csv', \n",
    "    'Friday-WorkingHours-Morning.pcap_ISCX.csv', \n",
    "    'Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv', \n",
    "    'Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv', \n",
    "    'Tuesday-WorkingHours.pcap_ISCX.csv', \n",
    "    'Wednesday-workingHours.pcap_ISCX.csv', \n",
    "    'Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "swedish-suspect",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_names = feature_map.values()\n",
    "df_out = pd.DataFrame(columns = feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "distinct-validity",
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in in_filenames:\n",
    "    df_in = pd.read_csv(indir + f)\n",
    "    \n",
    "    # Convert long column headers to compact format\n",
    "    df_in.rename(columns = feature_map, inplace=True)\n",
    "    \n",
    "    if f != 'Monday-WorkingHours.pcap_ISCX.csv':\n",
    "        df_in.drop(df_in[df_in.YY == 'BENIGN'].index, inplace=True)\n",
    "    \n",
    "    # Convert string labels to numeric\n",
    "    df_in['YY'].replace(label_map, inplace=True)\n",
    "    print (df_in.shape)\n",
    "    \n",
    "    # add to output dataframe\n",
    "    df_out = pd.concat([df_out, df_in], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "sweet-quarter",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_out.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fleet-egyptian",
   "metadata": {},
   "outputs": [],
   "source": [
    "# export to csv\n",
    "df_out.to_csv(outdir + mal_data, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8272b890",
   "metadata": {},
   "outputs": [],
   "source": [
    "1087564 - 529918"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}