autoencoder_power_system.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    },
    "colab": {
      "name": "autoencoder_power_system.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XUJDXR56gFgx"
      },
      "source": [
        "https://stackoverflow.com/questions/20906474/import-multiple-csv-files-into-pandas-and-concatenate-into-one-dataframe"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jxi9kvo1RyiD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c0b286a0-dd88-4a9a-d905-515884c292af"
      },
      "source": [
        "!pip install category_encoders"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting category_encoders\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/44/57/fcef41c248701ee62e8325026b90c432adea35555cbc870aff9cfba23727/category_encoders-2.2.2-py2.py3-none-any.whl (80kB)\n",
            "\r\u001b[K     |████                            | 10kB 15.8MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 20kB 2.8MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 30kB 3.9MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 40kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████████████████▎           | 51kB 3.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 61kB 3.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 71kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 81kB 3.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: statsmodels>=0.9.0 in /usr/local/lib/python3.6/dist-packages (from category_encoders) (0.10.2)\n",
            "Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from category_encoders) (1.4.1)\n",
            "Requirement already satisfied: pandas>=0.21.1 in /usr/local/lib/python3.6/dist-packages (from category_encoders) (1.1.4)\n",
            "Requirement already satisfied: patsy>=0.5.1 in /usr/local/lib/python3.6/dist-packages (from category_encoders) (0.5.1)\n",
            "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.6/dist-packages (from category_encoders) (0.22.2.post1)\n",
            "Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from category_encoders) (1.18.5)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.21.1->category_encoders) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.21.1->category_encoders) (2018.9)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from patsy>=0.5.1->category_encoders) (1.15.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.20.0->category_encoders) (0.17.0)\n",
            "Installing collected packages: category-encoders\n",
            "Successfully installed category-encoders-2.2.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SY6U1gHARWls"
      },
      "source": [
        "import numpy as np \n",
        "import tensorflow as tf \n",
        "import pandas as pd \n",
        "#import category_encoders as ce\n",
        "pd.options.mode.chained_assignment = None \n",
        "import seaborn as sns \n",
        "from matplotlib.pylab import rcParams \n",
        "import matplotlib.pyplot as plt\n",
        "import os\n",
        "# import plotly.express as px\n",
        "# import plotly.graph_objects as go\n",
        "\n",
        "# %matplotlib inline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oN6PnDLjRWl6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 131
        },
        "outputId": "b36a2be1-32bb-46d7-fff2-1945f24f8358"
      },
      "source": [
        "df1 = pd.read_csv('C:\\Users\\drish\\OneDrive\\Documents\\EXTC\\FYP\\DATASET\\binary_power_system\\data1.csv')\n",
        "df2 = pd.read_csv('C:\\Users\\drish\\OneDrive\\Documents\\EXTC\\FYP\\DATASET\\binary_power_system\\data2.csv')\n",
        "df3 = pd.read_csv('C:\\Users\\drish\\OneDrive\\Documents\\EXTC\\FYP\\DATASET\\binary_power_system\\data3.csv')\n",
        "df4 = pd.read_csv('C:\\Users\\drish\\OneDrive\\Documents\\EXTC\\FYP\\DATASET\\binary_power_system\\data4.csv')\n",
        "df5 = pd.read_csv('C:\\Users\\drish\\OneDrive\\Documents\\EXTC\\FYP\\DATASET\\binary_power_system\\data5.csv')\n",
        "\n",
        "df = df1.append([df2,df3,df4,df5])\n",
        "#from google.colab import files\n",
        "#uploaded = files.upload()\n",
        "df.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "SyntaxError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-2-48155245c373>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    df1 = pd.read_csv('C:\\Users\\drish\\OneDrive\\Documents\\EXTC\\FYP\\DATASET\\binary_power_system\\data1.csv')\u001b[0m\n\u001b[0m                     ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u1G7AjdY-AIj",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 361
        },
        "outputId": "cf615a6d-edb7-4efc-cb67-8af89b35c0a5"
      },
      "source": [
        "path = r'C:\\Users\\drish\\OneDrive\\Documents\\EXTC\\FYP\\DATASET\\binary_power_system'                     # use your path\n",
        "all_files = glob.glob(os.path.join(path, \"*.csv\"))     # advisable to use os.path.join as this makes concatenation OS independent\n",
        "\n",
        "df_from_each_file = (pd.read_csv(f) for f in all_files)\n",
        "concatenated_df   = pd.concat(df_from_each_file, ignore_index=True)\n",
        "# doesn't create a list, nor does it append to one"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-7-a63c77a0010f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mdf_from_each_file\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mall_files\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mconcatenated_df\u001b[0m   \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_from_each_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_index\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;31m# doesn't create a list, nor does it append to one\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py\u001b[0m in \u001b[0;36mconcat\u001b[0;34m(objs, axis, join, ignore_index, keys, levels, names, verify_integrity, sort, copy)\u001b[0m\n\u001b[1;32m    282\u001b[0m         \u001b[0mverify_integrity\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverify_integrity\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    283\u001b[0m         \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 284\u001b[0;31m         \u001b[0msort\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msort\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    285\u001b[0m     )\n\u001b[1;32m    286\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/reshape/concat.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, objs, axis, join, keys, levels, names, ignore_index, verify_integrity, copy, sort)\u001b[0m\n\u001b[1;32m    329\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    330\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobjs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 331\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"No objects to concatenate\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    332\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    333\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mkeys\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: No objects to concatenate"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oyP2DlzPjxit",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 303
        },
        "outputId": "3b057af2-dc0d-4824-de54-5ecce1e8d2cc"
      },
      "source": [
        "df = pd.read_csv('combined_binary.csv') #comment out this block if not uploading via computer\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127) have mixed types.Specify dtype option on import or set low_memory=False.\n",
            "  interactivity=interactivity, compiler=compiler, result=result)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>R1-PA1:VH</th>\n",
              "      <th>R1-PM1:V</th>\n",
              "      <th>R1-PA2:VH</th>\n",
              "      <th>R1-PM2:V</th>\n",
              "      <th>R1-PA3:VH</th>\n",
              "      <th>R1-PM3:V</th>\n",
              "      <th>R1-PA4:IH</th>\n",
              "      <th>R1-PM4:I</th>\n",
              "      <th>R1-PA5:IH</th>\n",
              "      <th>R1-PM5:I</th>\n",
              "      <th>R1-PA6:IH</th>\n",
              "      <th>R1-PM6:I</th>\n",
              "      <th>R1-PA7:VH</th>\n",
              "      <th>R1-PM7:V</th>\n",
              "      <th>R1-PA8:VH</th>\n",
              "      <th>R1-PM8:V</th>\n",
              "      <th>R1-PA9:VH</th>\n",
              "      <th>R1-PM9:V</th>\n",
              "      <th>R1-PA10:IH</th>\n",
              "      <th>R1-PM10:I</th>\n",
              "      <th>R1-PA11:IH</th>\n",
              "      <th>R1-PM11:I</th>\n",
              "      <th>R1-PA12:IH</th>\n",
              "      <th>R1-PM12:I</th>\n",
              "      <th>R1:F</th>\n",
              "      <th>R1:DF</th>\n",
              "      <th>R1-PA:Z</th>\n",
              "      <th>R1-PA:ZH</th>\n",
              "      <th>R1:S</th>\n",
              "      <th>R2-PA1:VH</th>\n",
              "      <th>R2-PM1:V</th>\n",
              "      <th>R2-PA2:VH</th>\n",
              "      <th>R2-PM2:V</th>\n",
              "      <th>R2-PA3:VH</th>\n",
              "      <th>R2-PM3:V</th>\n",
              "      <th>R2-PA4:IH</th>\n",
              "      <th>R2-PM4:I</th>\n",
              "      <th>R2-PA5:IH</th>\n",
              "      <th>R2-PM5:I</th>\n",
              "      <th>R2-PA6:IH</th>\n",
              "      <th>...</th>\n",
              "      <th>R4-PA2:VH</th>\n",
              "      <th>R4-PM2:V</th>\n",
              "      <th>R4-PA3:VH</th>\n",
              "      <th>R4-PM3:V</th>\n",
              "      <th>R4-PA4:IH</th>\n",
              "      <th>R4-PM4:I</th>\n",
              "      <th>R4-PA5:IH</th>\n",
              "      <th>R4-PM5:I</th>\n",
              "      <th>R4-PA6:IH</th>\n",
              "      <th>R4-PM6:I</th>\n",
              "      <th>R4-PA7:VH</th>\n",
              "      <th>R4-PM7:V</th>\n",
              "      <th>R4-PA8:VH</th>\n",
              "      <th>R4-PM8:V</th>\n",
              "      <th>R4-PA9:VH</th>\n",
              "      <th>R4-PM9:V</th>\n",
              "      <th>R4-PA10:IH</th>\n",
              "      <th>R4-PM10:I</th>\n",
              "      <th>R4-PA11:IH</th>\n",
              "      <th>R4-PM11:I</th>\n",
              "      <th>R4-PA12:IH</th>\n",
              "      <th>R4-PM12:I</th>\n",
              "      <th>R4:F</th>\n",
              "      <th>R4:DF</th>\n",
              "      <th>R4-PA:Z</th>\n",
              "      <th>R4-PA:ZH</th>\n",
              "      <th>R4:S</th>\n",
              "      <th>control_panel_log1</th>\n",
              "      <th>control_panel_log2</th>\n",
              "      <th>control_panel_log3</th>\n",
              "      <th>control_panel_log4</th>\n",
              "      <th>relay1_log</th>\n",
              "      <th>relay2_log</th>\n",
              "      <th>relay3_log</th>\n",
              "      <th>relay4_log</th>\n",
              "      <th>snort_log1</th>\n",
              "      <th>snort_log2</th>\n",
              "      <th>snort_log3</th>\n",
              "      <th>snort_log4</th>\n",
              "      <th>marker</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>70.3993</td>\n",
              "      <td>127673</td>\n",
              "      <td>-49.5723</td>\n",
              "      <td>127648</td>\n",
              "      <td>-169.578</td>\n",
              "      <td>127723</td>\n",
              "      <td>65.6896</td>\n",
              "      <td>605.911</td>\n",
              "      <td>-57.0036</td>\n",
              "      <td>626.786</td>\n",
              "      <td>-173.589</td>\n",
              "      <td>602.432</td>\n",
              "      <td>70.4222</td>\n",
              "      <td>127673</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>65.0078</td>\n",
              "      <td>611.587</td>\n",
              "      <td>118.568</td>\n",
              "      <td>13.1839</td>\n",
              "      <td>-100.869</td>\n",
              "      <td>13.9164</td>\n",
              "      <td>59.999</td>\n",
              "      <td>0.01</td>\n",
              "      <td>6.39138</td>\n",
              "      <td>0.0762905</td>\n",
              "      <td>0</td>\n",
              "      <td>60.6583</td>\n",
              "      <td>124632</td>\n",
              "      <td>-59.296</td>\n",
              "      <td>124484</td>\n",
              "      <td>-179.338</td>\n",
              "      <td>124715</td>\n",
              "      <td>-119.55</td>\n",
              "      <td>612.797</td>\n",
              "      <td>117.727</td>\n",
              "      <td>632.532</td>\n",
              "      <td>0.85968</td>\n",
              "      <td>...</td>\n",
              "      <td>-49.5379</td>\n",
              "      <td>127096</td>\n",
              "      <td>-169.532</td>\n",
              "      <td>127773</td>\n",
              "      <td>65.6438</td>\n",
              "      <td>604.446</td>\n",
              "      <td>-56.8718</td>\n",
              "      <td>621.842</td>\n",
              "      <td>-173.87</td>\n",
              "      <td>599.868</td>\n",
              "      <td>70.4623</td>\n",
              "      <td>127523</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>64.9505</td>\n",
              "      <td>608.475</td>\n",
              "      <td>119.301</td>\n",
              "      <td>12.2684</td>\n",
              "      <td>-102.061</td>\n",
              "      <td>11.719</td>\n",
              "      <td>59.999</td>\n",
              "      <td>0.01</td>\n",
              "      <td>6.34183</td>\n",
              "      <td>0.0778972</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>Natural</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>73.6881</td>\n",
              "      <td>130281</td>\n",
              "      <td>-46.3007</td>\n",
              "      <td>130256</td>\n",
              "      <td>-166.278</td>\n",
              "      <td>130356</td>\n",
              "      <td>71.8317</td>\n",
              "      <td>483.594</td>\n",
              "      <td>-50.9474</td>\n",
              "      <td>500.989</td>\n",
              "      <td>-167.487</td>\n",
              "      <td>481.396</td>\n",
              "      <td>73.7053</td>\n",
              "      <td>130306</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.1098</td>\n",
              "      <td>488.354</td>\n",
              "      <td>125.793</td>\n",
              "      <td>10.6204</td>\n",
              "      <td>-95.8845</td>\n",
              "      <td>11.3528</td>\n",
              "      <td>60.005</td>\n",
              "      <td>0</td>\n",
              "      <td>8.18546</td>\n",
              "      <td>0.0249238</td>\n",
              "      <td>0</td>\n",
              "      <td>66.069</td>\n",
              "      <td>128277</td>\n",
              "      <td>-53.8852</td>\n",
              "      <td>128126</td>\n",
              "      <td>-173.933</td>\n",
              "      <td>128355</td>\n",
              "      <td>-114.925</td>\n",
              "      <td>489.349</td>\n",
              "      <td>122.149</td>\n",
              "      <td>505.754</td>\n",
              "      <td>5.46844</td>\n",
              "      <td>...</td>\n",
              "      <td>-46.2492</td>\n",
              "      <td>129704</td>\n",
              "      <td>-166.232</td>\n",
              "      <td>130381</td>\n",
              "      <td>71.8374</td>\n",
              "      <td>481.762</td>\n",
              "      <td>-50.7927</td>\n",
              "      <td>496.045</td>\n",
              "      <td>-167.619</td>\n",
              "      <td>477.734</td>\n",
              "      <td>73.7569</td>\n",
              "      <td>130130</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.1213</td>\n",
              "      <td>485.058</td>\n",
              "      <td>124.275</td>\n",
              "      <td>10.2542</td>\n",
              "      <td>-95.4548</td>\n",
              "      <td>9.70483</td>\n",
              "      <td>60.005</td>\n",
              "      <td>0</td>\n",
              "      <td>8.14133</td>\n",
              "      <td>0.0272096</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>Natural</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>73.7339</td>\n",
              "      <td>130306</td>\n",
              "      <td>-46.2549</td>\n",
              "      <td>130281</td>\n",
              "      <td>-166.232</td>\n",
              "      <td>130381</td>\n",
              "      <td>71.8088</td>\n",
              "      <td>483.594</td>\n",
              "      <td>-50.913</td>\n",
              "      <td>500.989</td>\n",
              "      <td>-167.441</td>\n",
              "      <td>481.03</td>\n",
              "      <td>73.7511</td>\n",
              "      <td>130331</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.1327</td>\n",
              "      <td>488.354</td>\n",
              "      <td>125.008</td>\n",
              "      <td>10.6204</td>\n",
              "      <td>-94.5208</td>\n",
              "      <td>11.3528</td>\n",
              "      <td>60.005</td>\n",
              "      <td>0</td>\n",
              "      <td>8.19001</td>\n",
              "      <td>0.0279035</td>\n",
              "      <td>0</td>\n",
              "      <td>66.113</td>\n",
              "      <td>128284</td>\n",
              "      <td>-53.8522</td>\n",
              "      <td>128144</td>\n",
              "      <td>-173.883</td>\n",
              "      <td>128383</td>\n",
              "      <td>-114.92</td>\n",
              "      <td>488.886</td>\n",
              "      <td>122.215</td>\n",
              "      <td>505.531</td>\n",
              "      <td>5.46295</td>\n",
              "      <td>...</td>\n",
              "      <td>-46.1976</td>\n",
              "      <td>129729</td>\n",
              "      <td>-166.192</td>\n",
              "      <td>130381</td>\n",
              "      <td>71.8661</td>\n",
              "      <td>481.396</td>\n",
              "      <td>-50.7812</td>\n",
              "      <td>496.411</td>\n",
              "      <td>-167.59</td>\n",
              "      <td>478.1</td>\n",
              "      <td>73.797</td>\n",
              "      <td>130155</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.1442</td>\n",
              "      <td>485.241</td>\n",
              "      <td>125.277</td>\n",
              "      <td>10.2542</td>\n",
              "      <td>-95.9704</td>\n",
              "      <td>10.071</td>\n",
              "      <td>60.005</td>\n",
              "      <td>0</td>\n",
              "      <td>8.16276</td>\n",
              "      <td>0.0266635</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>Natural</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>74.0834</td>\n",
              "      <td>130582</td>\n",
              "      <td>-45.8996</td>\n",
              "      <td>130557</td>\n",
              "      <td>-165.883</td>\n",
              "      <td>130657</td>\n",
              "      <td>72.1526</td>\n",
              "      <td>482.861</td>\n",
              "      <td>-50.4375</td>\n",
              "      <td>499.158</td>\n",
              "      <td>-167.286</td>\n",
              "      <td>481.396</td>\n",
              "      <td>74.1064</td>\n",
              "      <td>130582</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.4593</td>\n",
              "      <td>487.622</td>\n",
              "      <td>127.598</td>\n",
              "      <td>9.70483</td>\n",
              "      <td>-96.658</td>\n",
              "      <td>10.4373</td>\n",
              "      <td>60.003</td>\n",
              "      <td>0</td>\n",
              "      <td>8.17153</td>\n",
              "      <td>0.025617</td>\n",
              "      <td>0</td>\n",
              "      <td>66.5085</td>\n",
              "      <td>128585</td>\n",
              "      <td>-53.4457</td>\n",
              "      <td>128443</td>\n",
              "      <td>-173.488</td>\n",
              "      <td>128673</td>\n",
              "      <td>-114.815</td>\n",
              "      <td>489.864</td>\n",
              "      <td>122.649</td>\n",
              "      <td>504.227</td>\n",
              "      <td>5.69366</td>\n",
              "      <td>...</td>\n",
              "      <td>-45.8481</td>\n",
              "      <td>129980</td>\n",
              "      <td>-165.837</td>\n",
              "      <td>130682</td>\n",
              "      <td>72.1812</td>\n",
              "      <td>481.213</td>\n",
              "      <td>-50.3114</td>\n",
              "      <td>494.763</td>\n",
              "      <td>-167.395</td>\n",
              "      <td>478.283</td>\n",
              "      <td>74.1522</td>\n",
              "      <td>130431</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.4822</td>\n",
              "      <td>484.692</td>\n",
              "      <td>126.475</td>\n",
              "      <td>9.33861</td>\n",
              "      <td>-97.2539</td>\n",
              "      <td>9.1555</td>\n",
              "      <td>60.003</td>\n",
              "      <td>0</td>\n",
              "      <td>8.13529</td>\n",
              "      <td>0.0265955</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>Natural</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>74.5533</td>\n",
              "      <td>131083</td>\n",
              "      <td>-45.4241</td>\n",
              "      <td>131058</td>\n",
              "      <td>-165.424</td>\n",
              "      <td>131158</td>\n",
              "      <td>72.1182</td>\n",
              "      <td>484.509</td>\n",
              "      <td>-50.0135</td>\n",
              "      <td>497.693</td>\n",
              "      <td>-167.464</td>\n",
              "      <td>484.692</td>\n",
              "      <td>74.5705</td>\n",
              "      <td>131108</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.5395</td>\n",
              "      <td>488.904</td>\n",
              "      <td>127.666</td>\n",
              "      <td>7.50751</td>\n",
              "      <td>-99.9238</td>\n",
              "      <td>8.60617</td>\n",
              "      <td>60.001</td>\n",
              "      <td>0</td>\n",
              "      <td>8.0795</td>\n",
              "      <td>0.0329408</td>\n",
              "      <td>0</td>\n",
              "      <td>67.0358</td>\n",
              "      <td>129107</td>\n",
              "      <td>-52.9129</td>\n",
              "      <td>128975</td>\n",
              "      <td>-172.95</td>\n",
              "      <td>129197</td>\n",
              "      <td>-114.92</td>\n",
              "      <td>491.295</td>\n",
              "      <td>123.072</td>\n",
              "      <td>502.499</td>\n",
              "      <td>5.41351</td>\n",
              "      <td>...</td>\n",
              "      <td>-45.3725</td>\n",
              "      <td>130506</td>\n",
              "      <td>-165.373</td>\n",
              "      <td>131183</td>\n",
              "      <td>72.164</td>\n",
              "      <td>482.861</td>\n",
              "      <td>-49.8817</td>\n",
              "      <td>493.115</td>\n",
              "      <td>-167.602</td>\n",
              "      <td>481.213</td>\n",
              "      <td>74.6278</td>\n",
              "      <td>130933</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.5567</td>\n",
              "      <td>485.608</td>\n",
              "      <td>126.595</td>\n",
              "      <td>7.3244</td>\n",
              "      <td>-101.711</td>\n",
              "      <td>7.14129</td>\n",
              "      <td>60.001</td>\n",
              "      <td>0</td>\n",
              "      <td>8.03602</td>\n",
              "      <td>0.0336413</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>Natural</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 129 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "  R1-PA1:VH R1-PM1:V R1-PA2:VH  ... snort_log3 snort_log4   marker\n",
              "0   70.3993   127673  -49.5723  ...          0          0  Natural\n",
              "1   73.6881   130281  -46.3007  ...          0          0  Natural\n",
              "2   73.7339   130306  -46.2549  ...          0          0  Natural\n",
              "3   74.0834   130582  -45.8996  ...          0          0  Natural\n",
              "4   74.5533   131083  -45.4241  ...          0          0  Natural\n",
              "\n",
              "[5 rows x 129 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XJPqQiGtRWmC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5daa7eea-e891-4c3b-d370-b104e26c23f9"
      },
      "source": [
        "df.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(78392, 129)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LISCTRmDRWmI",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "outputId": "95c6bd40-7fc5-4046-9b1e-b7d768edb3df"
      },
      "source": [
        "X1 = df.drop(['marker'],axis = 1,inplace=False)\n",
        "y = df.marker\n",
        "X1.head()\n",
        "#print(X1.max())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>R1-PA1:VH</th>\n",
              "      <th>R1-PM1:V</th>\n",
              "      <th>R1-PA2:VH</th>\n",
              "      <th>R1-PM2:V</th>\n",
              "      <th>R1-PA3:VH</th>\n",
              "      <th>R1-PM3:V</th>\n",
              "      <th>R1-PA4:IH</th>\n",
              "      <th>R1-PM4:I</th>\n",
              "      <th>R1-PA5:IH</th>\n",
              "      <th>R1-PM5:I</th>\n",
              "      <th>R1-PA6:IH</th>\n",
              "      <th>R1-PM6:I</th>\n",
              "      <th>R1-PA7:VH</th>\n",
              "      <th>R1-PM7:V</th>\n",
              "      <th>R1-PA8:VH</th>\n",
              "      <th>R1-PM8:V</th>\n",
              "      <th>R1-PA9:VH</th>\n",
              "      <th>R1-PM9:V</th>\n",
              "      <th>R1-PA10:IH</th>\n",
              "      <th>R1-PM10:I</th>\n",
              "      <th>R1-PA11:IH</th>\n",
              "      <th>R1-PM11:I</th>\n",
              "      <th>R1-PA12:IH</th>\n",
              "      <th>R1-PM12:I</th>\n",
              "      <th>R1:F</th>\n",
              "      <th>R1:DF</th>\n",
              "      <th>R1-PA:Z</th>\n",
              "      <th>R1-PA:ZH</th>\n",
              "      <th>R1:S</th>\n",
              "      <th>R2-PA1:VH</th>\n",
              "      <th>R2-PM1:V</th>\n",
              "      <th>R2-PA2:VH</th>\n",
              "      <th>R2-PM2:V</th>\n",
              "      <th>R2-PA3:VH</th>\n",
              "      <th>R2-PM3:V</th>\n",
              "      <th>R2-PA4:IH</th>\n",
              "      <th>R2-PM4:I</th>\n",
              "      <th>R2-PA5:IH</th>\n",
              "      <th>R2-PM5:I</th>\n",
              "      <th>R2-PA6:IH</th>\n",
              "      <th>...</th>\n",
              "      <th>R4-PM1:V</th>\n",
              "      <th>R4-PA2:VH</th>\n",
              "      <th>R4-PM2:V</th>\n",
              "      <th>R4-PA3:VH</th>\n",
              "      <th>R4-PM3:V</th>\n",
              "      <th>R4-PA4:IH</th>\n",
              "      <th>R4-PM4:I</th>\n",
              "      <th>R4-PA5:IH</th>\n",
              "      <th>R4-PM5:I</th>\n",
              "      <th>R4-PA6:IH</th>\n",
              "      <th>R4-PM6:I</th>\n",
              "      <th>R4-PA7:VH</th>\n",
              "      <th>R4-PM7:V</th>\n",
              "      <th>R4-PA8:VH</th>\n",
              "      <th>R4-PM8:V</th>\n",
              "      <th>R4-PA9:VH</th>\n",
              "      <th>R4-PM9:V</th>\n",
              "      <th>R4-PA10:IH</th>\n",
              "      <th>R4-PM10:I</th>\n",
              "      <th>R4-PA11:IH</th>\n",
              "      <th>R4-PM11:I</th>\n",
              "      <th>R4-PA12:IH</th>\n",
              "      <th>R4-PM12:I</th>\n",
              "      <th>R4:F</th>\n",
              "      <th>R4:DF</th>\n",
              "      <th>R4-PA:Z</th>\n",
              "      <th>R4-PA:ZH</th>\n",
              "      <th>R4:S</th>\n",
              "      <th>control_panel_log1</th>\n",
              "      <th>control_panel_log2</th>\n",
              "      <th>control_panel_log3</th>\n",
              "      <th>control_panel_log4</th>\n",
              "      <th>relay1_log</th>\n",
              "      <th>relay2_log</th>\n",
              "      <th>relay3_log</th>\n",
              "      <th>relay4_log</th>\n",
              "      <th>snort_log1</th>\n",
              "      <th>snort_log2</th>\n",
              "      <th>snort_log3</th>\n",
              "      <th>snort_log4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>70.3993</td>\n",
              "      <td>127673</td>\n",
              "      <td>-49.5723</td>\n",
              "      <td>127648</td>\n",
              "      <td>-169.578</td>\n",
              "      <td>127723</td>\n",
              "      <td>65.6896</td>\n",
              "      <td>605.911</td>\n",
              "      <td>-57.0036</td>\n",
              "      <td>626.786</td>\n",
              "      <td>-173.589</td>\n",
              "      <td>602.432</td>\n",
              "      <td>70.4222</td>\n",
              "      <td>127673</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>65.0078</td>\n",
              "      <td>611.587</td>\n",
              "      <td>118.568</td>\n",
              "      <td>13.1839</td>\n",
              "      <td>-100.869</td>\n",
              "      <td>13.9164</td>\n",
              "      <td>59.999</td>\n",
              "      <td>0.01</td>\n",
              "      <td>6.39138</td>\n",
              "      <td>0.0762905</td>\n",
              "      <td>0</td>\n",
              "      <td>60.6583</td>\n",
              "      <td>124632</td>\n",
              "      <td>-59.296</td>\n",
              "      <td>124484</td>\n",
              "      <td>-179.338</td>\n",
              "      <td>124715</td>\n",
              "      <td>-119.55</td>\n",
              "      <td>612.797</td>\n",
              "      <td>117.727</td>\n",
              "      <td>632.532</td>\n",
              "      <td>0.85968</td>\n",
              "      <td>...</td>\n",
              "      <td>127723</td>\n",
              "      <td>-49.5379</td>\n",
              "      <td>127096</td>\n",
              "      <td>-169.532</td>\n",
              "      <td>127773</td>\n",
              "      <td>65.6438</td>\n",
              "      <td>604.446</td>\n",
              "      <td>-56.8718</td>\n",
              "      <td>621.842</td>\n",
              "      <td>-173.87</td>\n",
              "      <td>599.868</td>\n",
              "      <td>70.4623</td>\n",
              "      <td>127523</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>64.9505</td>\n",
              "      <td>608.475</td>\n",
              "      <td>119.301</td>\n",
              "      <td>12.2684</td>\n",
              "      <td>-102.061</td>\n",
              "      <td>11.719</td>\n",
              "      <td>59.999</td>\n",
              "      <td>0.01</td>\n",
              "      <td>6.34183</td>\n",
              "      <td>0.0778972</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>73.6881</td>\n",
              "      <td>130281</td>\n",
              "      <td>-46.3007</td>\n",
              "      <td>130256</td>\n",
              "      <td>-166.278</td>\n",
              "      <td>130356</td>\n",
              "      <td>71.8317</td>\n",
              "      <td>483.594</td>\n",
              "      <td>-50.9474</td>\n",
              "      <td>500.989</td>\n",
              "      <td>-167.487</td>\n",
              "      <td>481.396</td>\n",
              "      <td>73.7053</td>\n",
              "      <td>130306</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.1098</td>\n",
              "      <td>488.354</td>\n",
              "      <td>125.793</td>\n",
              "      <td>10.6204</td>\n",
              "      <td>-95.8845</td>\n",
              "      <td>11.3528</td>\n",
              "      <td>60.005</td>\n",
              "      <td>0</td>\n",
              "      <td>8.18546</td>\n",
              "      <td>0.0249238</td>\n",
              "      <td>0</td>\n",
              "      <td>66.069</td>\n",
              "      <td>128277</td>\n",
              "      <td>-53.8852</td>\n",
              "      <td>128126</td>\n",
              "      <td>-173.933</td>\n",
              "      <td>128355</td>\n",
              "      <td>-114.925</td>\n",
              "      <td>489.349</td>\n",
              "      <td>122.149</td>\n",
              "      <td>505.754</td>\n",
              "      <td>5.46844</td>\n",
              "      <td>...</td>\n",
              "      <td>130331</td>\n",
              "      <td>-46.2492</td>\n",
              "      <td>129704</td>\n",
              "      <td>-166.232</td>\n",
              "      <td>130381</td>\n",
              "      <td>71.8374</td>\n",
              "      <td>481.762</td>\n",
              "      <td>-50.7927</td>\n",
              "      <td>496.045</td>\n",
              "      <td>-167.619</td>\n",
              "      <td>477.734</td>\n",
              "      <td>73.7569</td>\n",
              "      <td>130130</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.1213</td>\n",
              "      <td>485.058</td>\n",
              "      <td>124.275</td>\n",
              "      <td>10.2542</td>\n",
              "      <td>-95.4548</td>\n",
              "      <td>9.70483</td>\n",
              "      <td>60.005</td>\n",
              "      <td>0</td>\n",
              "      <td>8.14133</td>\n",
              "      <td>0.0272096</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>73.7339</td>\n",
              "      <td>130306</td>\n",
              "      <td>-46.2549</td>\n",
              "      <td>130281</td>\n",
              "      <td>-166.232</td>\n",
              "      <td>130381</td>\n",
              "      <td>71.8088</td>\n",
              "      <td>483.594</td>\n",
              "      <td>-50.913</td>\n",
              "      <td>500.989</td>\n",
              "      <td>-167.441</td>\n",
              "      <td>481.03</td>\n",
              "      <td>73.7511</td>\n",
              "      <td>130331</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.1327</td>\n",
              "      <td>488.354</td>\n",
              "      <td>125.008</td>\n",
              "      <td>10.6204</td>\n",
              "      <td>-94.5208</td>\n",
              "      <td>11.3528</td>\n",
              "      <td>60.005</td>\n",
              "      <td>0</td>\n",
              "      <td>8.19001</td>\n",
              "      <td>0.0279035</td>\n",
              "      <td>0</td>\n",
              "      <td>66.113</td>\n",
              "      <td>128284</td>\n",
              "      <td>-53.8522</td>\n",
              "      <td>128144</td>\n",
              "      <td>-173.883</td>\n",
              "      <td>128383</td>\n",
              "      <td>-114.92</td>\n",
              "      <td>488.886</td>\n",
              "      <td>122.215</td>\n",
              "      <td>505.531</td>\n",
              "      <td>5.46295</td>\n",
              "      <td>...</td>\n",
              "      <td>130356</td>\n",
              "      <td>-46.1976</td>\n",
              "      <td>129729</td>\n",
              "      <td>-166.192</td>\n",
              "      <td>130381</td>\n",
              "      <td>71.8661</td>\n",
              "      <td>481.396</td>\n",
              "      <td>-50.7812</td>\n",
              "      <td>496.411</td>\n",
              "      <td>-167.59</td>\n",
              "      <td>478.1</td>\n",
              "      <td>73.797</td>\n",
              "      <td>130155</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.1442</td>\n",
              "      <td>485.241</td>\n",
              "      <td>125.277</td>\n",
              "      <td>10.2542</td>\n",
              "      <td>-95.9704</td>\n",
              "      <td>10.071</td>\n",
              "      <td>60.005</td>\n",
              "      <td>0</td>\n",
              "      <td>8.16276</td>\n",
              "      <td>0.0266635</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>74.0834</td>\n",
              "      <td>130582</td>\n",
              "      <td>-45.8996</td>\n",
              "      <td>130557</td>\n",
              "      <td>-165.883</td>\n",
              "      <td>130657</td>\n",
              "      <td>72.1526</td>\n",
              "      <td>482.861</td>\n",
              "      <td>-50.4375</td>\n",
              "      <td>499.158</td>\n",
              "      <td>-167.286</td>\n",
              "      <td>481.396</td>\n",
              "      <td>74.1064</td>\n",
              "      <td>130582</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.4593</td>\n",
              "      <td>487.622</td>\n",
              "      <td>127.598</td>\n",
              "      <td>9.70483</td>\n",
              "      <td>-96.658</td>\n",
              "      <td>10.4373</td>\n",
              "      <td>60.003</td>\n",
              "      <td>0</td>\n",
              "      <td>8.17153</td>\n",
              "      <td>0.025617</td>\n",
              "      <td>0</td>\n",
              "      <td>66.5085</td>\n",
              "      <td>128585</td>\n",
              "      <td>-53.4457</td>\n",
              "      <td>128443</td>\n",
              "      <td>-173.488</td>\n",
              "      <td>128673</td>\n",
              "      <td>-114.815</td>\n",
              "      <td>489.864</td>\n",
              "      <td>122.649</td>\n",
              "      <td>504.227</td>\n",
              "      <td>5.69366</td>\n",
              "      <td>...</td>\n",
              "      <td>130632</td>\n",
              "      <td>-45.8481</td>\n",
              "      <td>129980</td>\n",
              "      <td>-165.837</td>\n",
              "      <td>130682</td>\n",
              "      <td>72.1812</td>\n",
              "      <td>481.213</td>\n",
              "      <td>-50.3114</td>\n",
              "      <td>494.763</td>\n",
              "      <td>-167.395</td>\n",
              "      <td>478.283</td>\n",
              "      <td>74.1522</td>\n",
              "      <td>130431</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.4822</td>\n",
              "      <td>484.692</td>\n",
              "      <td>126.475</td>\n",
              "      <td>9.33861</td>\n",
              "      <td>-97.2539</td>\n",
              "      <td>9.1555</td>\n",
              "      <td>60.003</td>\n",
              "      <td>0</td>\n",
              "      <td>8.13529</td>\n",
              "      <td>0.0265955</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>74.5533</td>\n",
              "      <td>131083</td>\n",
              "      <td>-45.4241</td>\n",
              "      <td>131058</td>\n",
              "      <td>-165.424</td>\n",
              "      <td>131158</td>\n",
              "      <td>72.1182</td>\n",
              "      <td>484.509</td>\n",
              "      <td>-50.0135</td>\n",
              "      <td>497.693</td>\n",
              "      <td>-167.464</td>\n",
              "      <td>484.692</td>\n",
              "      <td>74.5705</td>\n",
              "      <td>131108</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.5395</td>\n",
              "      <td>488.904</td>\n",
              "      <td>127.666</td>\n",
              "      <td>7.50751</td>\n",
              "      <td>-99.9238</td>\n",
              "      <td>8.60617</td>\n",
              "      <td>60.001</td>\n",
              "      <td>0</td>\n",
              "      <td>8.0795</td>\n",
              "      <td>0.0329408</td>\n",
              "      <td>0</td>\n",
              "      <td>67.0358</td>\n",
              "      <td>129107</td>\n",
              "      <td>-52.9129</td>\n",
              "      <td>128975</td>\n",
              "      <td>-172.95</td>\n",
              "      <td>129197</td>\n",
              "      <td>-114.92</td>\n",
              "      <td>491.295</td>\n",
              "      <td>123.072</td>\n",
              "      <td>502.499</td>\n",
              "      <td>5.41351</td>\n",
              "      <td>...</td>\n",
              "      <td>131133</td>\n",
              "      <td>-45.3725</td>\n",
              "      <td>130506</td>\n",
              "      <td>-165.373</td>\n",
              "      <td>131183</td>\n",
              "      <td>72.164</td>\n",
              "      <td>482.861</td>\n",
              "      <td>-49.8817</td>\n",
              "      <td>493.115</td>\n",
              "      <td>-167.602</td>\n",
              "      <td>481.213</td>\n",
              "      <td>74.6278</td>\n",
              "      <td>130933</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>71.5567</td>\n",
              "      <td>485.608</td>\n",
              "      <td>126.595</td>\n",
              "      <td>7.3244</td>\n",
              "      <td>-101.711</td>\n",
              "      <td>7.14129</td>\n",
              "      <td>60.001</td>\n",
              "      <td>0</td>\n",
              "      <td>8.03602</td>\n",
              "      <td>0.0336413</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 128 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "  R1-PA1:VH R1-PM1:V R1-PA2:VH  ... snort_log2 snort_log3 snort_log4\n",
              "0   70.3993   127673  -49.5723  ...          0          0          0\n",
              "1   73.6881   130281  -46.3007  ...          0          0          0\n",
              "2   73.7339   130306  -46.2549  ...          0          0          0\n",
              "3   74.0834   130582  -45.8996  ...          0          0          0\n",
              "4   74.5533   131083  -45.4241  ...          0          0          0\n",
              "\n",
              "[5 rows x 128 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h_2CGpTeRWmP",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "outputId": "cb1f9c0b-28df-439b-869d-e31a18faf313"
      },
      "source": [
        "X = X1 / X1.max() #Normalization\n",
        "X.head()\n",
        "#X.tail()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>R1-PA1:VH</th>\n",
              "      <th>R1-PM1:V</th>\n",
              "      <th>R1-PA2:VH</th>\n",
              "      <th>R1-PM2:V</th>\n",
              "      <th>R1-PA3:VH</th>\n",
              "      <th>R1-PM3:V</th>\n",
              "      <th>R1-PA4:IH</th>\n",
              "      <th>R1-PM4:I</th>\n",
              "      <th>R1-PA5:IH</th>\n",
              "      <th>R1-PM5:I</th>\n",
              "      <th>R1-PA6:IH</th>\n",
              "      <th>R1-PM6:I</th>\n",
              "      <th>R1-PA7:VH</th>\n",
              "      <th>R1-PM7:V</th>\n",
              "      <th>R1-PA8:VH</th>\n",
              "      <th>R1-PM8:V</th>\n",
              "      <th>R1-PA9:VH</th>\n",
              "      <th>R1-PM9:V</th>\n",
              "      <th>R1-PA10:IH</th>\n",
              "      <th>R1-PM10:I</th>\n",
              "      <th>R1-PA11:IH</th>\n",
              "      <th>R1-PM11:I</th>\n",
              "      <th>R1-PA12:IH</th>\n",
              "      <th>R1-PM12:I</th>\n",
              "      <th>R1:F</th>\n",
              "      <th>R1:DF</th>\n",
              "      <th>R1-PA:Z</th>\n",
              "      <th>R1-PA:ZH</th>\n",
              "      <th>R1:S</th>\n",
              "      <th>R2-PA1:VH</th>\n",
              "      <th>R2-PM1:V</th>\n",
              "      <th>R2-PA2:VH</th>\n",
              "      <th>R2-PM2:V</th>\n",
              "      <th>R2-PA3:VH</th>\n",
              "      <th>R2-PM3:V</th>\n",
              "      <th>R2-PA4:IH</th>\n",
              "      <th>R2-PM4:I</th>\n",
              "      <th>R2-PA5:IH</th>\n",
              "      <th>R2-PM5:I</th>\n",
              "      <th>R2-PA6:IH</th>\n",
              "      <th>...</th>\n",
              "      <th>R4-PM1:V</th>\n",
              "      <th>R4-PA2:VH</th>\n",
              "      <th>R4-PM2:V</th>\n",
              "      <th>R4-PA3:VH</th>\n",
              "      <th>R4-PM3:V</th>\n",
              "      <th>R4-PA4:IH</th>\n",
              "      <th>R4-PM4:I</th>\n",
              "      <th>R4-PA5:IH</th>\n",
              "      <th>R4-PM5:I</th>\n",
              "      <th>R4-PA6:IH</th>\n",
              "      <th>R4-PM6:I</th>\n",
              "      <th>R4-PA7:VH</th>\n",
              "      <th>R4-PM7:V</th>\n",
              "      <th>R4-PA8:VH</th>\n",
              "      <th>R4-PM8:V</th>\n",
              "      <th>R4-PA9:VH</th>\n",
              "      <th>R4-PM9:V</th>\n",
              "      <th>R4-PA10:IH</th>\n",
              "      <th>R4-PM10:I</th>\n",
              "      <th>R4-PA11:IH</th>\n",
              "      <th>R4-PM11:I</th>\n",
              "      <th>R4-PA12:IH</th>\n",
              "      <th>R4-PM12:I</th>\n",
              "      <th>R4:F</th>\n",
              "      <th>R4:DF</th>\n",
              "      <th>R4-PA:Z</th>\n",
              "      <th>R4-PA:ZH</th>\n",
              "      <th>R4:S</th>\n",
              "      <th>control_panel_log1</th>\n",
              "      <th>control_panel_log2</th>\n",
              "      <th>control_panel_log3</th>\n",
              "      <th>control_panel_log4</th>\n",
              "      <th>relay1_log</th>\n",
              "      <th>relay2_log</th>\n",
              "      <th>relay3_log</th>\n",
              "      <th>relay4_log</th>\n",
              "      <th>snort_log1</th>\n",
              "      <th>snort_log2</th>\n",
              "      <th>snort_log3</th>\n",
              "      <th>snort_log4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>78387</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>78388</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>78389</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>78390</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>78391</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 128 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      R1-PA1:VH R1-PM1:V R1-PA2:VH  ... snort_log2 snort_log3 snort_log4\n",
              "78387       NaN      NaN       NaN  ...        NaN        NaN        NaN\n",
              "78388       NaN      NaN       NaN  ...        NaN        NaN        NaN\n",
              "78389       NaN      NaN       NaN  ...        NaN        NaN        NaN\n",
              "78390       NaN      NaN       NaN  ...        NaN        NaN        NaN\n",
              "78391       NaN      NaN       NaN  ...        NaN        NaN        NaN\n",
              "\n",
              "[5 rows x 128 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-quCGfIuRWmW"
      },
      "source": [
        "X.dropna(axis = 1,inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sTykxgq5RWmb"
      },
      "source": [
        "assert not np.any(np.isnan(X))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W8lzhQqJRWmi",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 430
        },
        "outputId": "c09a5af6-ae21-4b7a-9103-dd2d4db9ed88"
      },
      "source": [
        "from sklearn.preprocessing import LabelEncoder\n",
        "le = LabelEncoder()\n",
        "y = le.fit_transform(y)\n",
        "y\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/preprocessing/_label.py\u001b[0m in \u001b[0;36m_encode\u001b[0;34m(values, uniques, encode, check_unknown)\u001b[0m\n\u001b[1;32m    111\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m             \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_encode_python\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muniques\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    113\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/preprocessing/_label.py\u001b[0m in \u001b[0;36m_encode_python\u001b[0;34m(values, uniques, encode)\u001b[0m\n\u001b[1;32m     59\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0muniques\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m         \u001b[0muniques\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     61\u001b[0m         \u001b[0muniques\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0muniques\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: '<' not supported between instances of 'str' and 'float'",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-13-3a5663ccd234>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreprocessing\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLabelEncoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mle\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLabelEncoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/preprocessing/_label.py\u001b[0m in \u001b[0;36mfit_transform\u001b[0;34m(self, y)\u001b[0m\n\u001b[1;32m    250\u001b[0m         \"\"\"\n\u001b[1;32m    251\u001b[0m         \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcolumn_or_1d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwarn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 252\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_encode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    253\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    254\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/sklearn/preprocessing/_label.py\u001b[0m in \u001b[0;36m_encode\u001b[0;34m(values, uniques, encode, check_unknown)\u001b[0m\n\u001b[1;32m    112\u001b[0m             \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_encode_python\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muniques\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"argument must be a string or number\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    115\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: argument must be a string or number"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OyV5y8iaRWmo",
        "outputId": "cc42037f-8de2-4dbb-f8c9-0c80aefdd8eb"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "np.random.seed(7)\n",
        "(X_train, X_test, y_train, y_test) = train_test_split(X,y,test_size=0.1, random_state=42)\n",
        "print(X_train.shape,\" \",y_test.shape)\n",
        "input_shape = X_train.shape[1]\n",
        "print(input_shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(23231, 120)   (2582,)\n",
            "120\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cgMSbkVRRWmu",
        "outputId": "df840117-622d-4dbb-cf25-0232fee6fc73"
      },
      "source": [
        "import seaborn as sns\n",
        "ax = sns.countplot(x='marker',data=df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEGCAYAAACkQqisAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAWgklEQVR4nO3dfbRddX3n8ffHIGpVFCQySKCJTnQGqQZJkdbqMKIQmGrA+kBcLVGZibikHR3tiJ21CkuL41gcO1qKCzUFZimIUiQ6KEbG0fEBSajIg0qJQOGaTIjEVnxYdEW/88f53bK5nJtcdu49h3Dfr7XOOnt/98P5bdYJn7v3b+/fSVUhSVIfjxp3AyRJey5DRJLUmyEiSerNEJEk9WaISJJ622vcDRi1/fffvxYvXjzuZkjSHuW66677UVUtnFqfdyGyePFiNm7cOO5mSNIeJcnfD6t7OUuS1JshIknqzRCRJPVmiEiSejNEJEm9GSKSpN4MEUlSb4aIJKk3Q0SS1Nu8e2JdeiS7812/Me4m6GHokD+9cc727ZmIJKk3Q0SS1JshIknqzRCRJPU2ZyGSZG2Su5Pc1Kl9Msn17XVHkutbfXGSX3SWfbizzRFJbkyyKckHk6TV90uyPsmt7X3fuToWSdJwc3kmcgGwoluoqtdU1bKqWgZcBvxNZ/EPJpdV1Wmd+nnAGmBpe03u8wzg6qpaClzd5iVJIzRnIVJVXwW2D1vWziZeDVy8s30kORDYp6q+WVUFXASc2BavBC5s0xd26pKkERlXn8gLga1VdWuntiTJt5N8JckLW+0gYKKzzkSrARxQVVsA2vtT57rRkqQHGtfDhqt44FnIFuCQqronyRHAZ5I8G8iQbeuhfliSNQwuiXHIIYf0aK4kaZiRn4kk2Qt4BfDJyVpV3VdV97Tp64AfAM9kcOaxqLP5ImBzm97aLndNXva6e7rPrKrzq2p5VS1fuPBBvzMvSeppHJezXgJ8v6r++TJVkoVJFrTppzPoQL+tXaa6N8lRrR/lFOCKttk6YHWbXt2pS5JGZC5v8b0Y+CbwrCQTSU5ti07mwR3qLwJuSPId4NPAaVU12Sn/JuCjwCYGZyifb/X3Ai9Ncivw0jYvSRqhOesTqapV09RfN6R2GYNbfoetvxE4bEj9HuCY3WulJGl3+MS6JKk3Q0SS1JshIknqzRCRJPVmiEiSejNEJEm9GSKSpN4MEUlSb4aIJKk3Q0SS1JshIknqzRCRJPVmiEiSejNEJEm9GSKSpN4MEUlSb4aIJKk3Q0SS1JshIknqzRCRJPU2ZyGSZG2Su5Pc1KmdleSHSa5vrxM6y96ZZFOSW5Ic16mvaLVNSc7o1Jck+VaSW5N8Msnec3UskqTh5vJM5AJgxZD6B6pqWXtdCZDkUOBk4Nltm79KsiDJAuBc4HjgUGBVWxfgv7V9LQV+DJw6h8ciSRpizkKkqr4KbJ/h6iuBS6rqvqq6HdgEHNlem6rqtqr6J+ASYGWSAC8GPt22vxA4cVYPQJK0S+PoEzk9yQ3tcte+rXYQcFdnnYlWm67+FOAfqmrHlPpQSdYk2Zhk47Zt22brOCRp3ht1iJwHPANYBmwB3t/qGbJu9agPVVXnV9Xyqlq+cOHCh9ZiSdK09hrlh1XV1snpJB8BPtdmJ4CDO6suAja36WH1HwFPTrJXOxvpri9JGpGRnokkObAzexIweefWOuDkJI9JsgRYClwLbACWtjux9mbQ+b6uqgr4MvDKtv1q4IpRHIMk6X5zdiaS5GLgaGD/JBPAmcDRSZYxuPR0B/BGgKq6OcmlwHeBHcCbq+qXbT+nA1cBC4C1VXVz+4h3AJck+TPg28DH5upYJEnDzVmIVNWqIeVp/0dfVWcDZw+pXwlcOaR+G4O7tyRJY+IT65Kk3gwRSVJvhogkqTdDRJLUmyEiSerNEJEk9WaISJJ6M0QkSb0ZIpKk3gwRSVJvhogkqTdDRJLUmyEiSerNEJEk9WaISJJ6M0QkSb0ZIpKk3gwRSVJvhogkqbc5C5Eka5PcneSmTu3Pk3w/yQ1JLk/y5FZfnOQXSa5vrw93tjkiyY1JNiX5YJK0+n5J1ie5tb3vO1fHIkkabi7PRC4AVkyprQcOq6rnAH8HvLOz7AdVtay9TuvUzwPWAEvba3KfZwBXV9VS4Oo2L0kaoTkLkar6KrB9Su2LVbWjzV4DLNrZPpIcCOxTVd+sqgIuAk5si1cCF7bpCzt1SdKIjLNP5A3A5zvzS5J8O8lXkryw1Q4CJjrrTLQawAFVtQWgvT91ug9KsibJxiQbt23bNntHIEnz3FhCJMl/AXYAH2+lLcAhVXU48J+ATyTZB8iQzeuhfl5VnV9Vy6tq+cKFC/s2W5I0xV6j/sAkq4HfBY5pl6ioqvuA+9r0dUl+ADyTwZlH95LXImBzm96a5MCq2tIue909qmOQJA2M9EwkyQrgHcDLq+rnnfrCJAva9NMZdKDf1i5T3ZvkqHZX1inAFW2zdcDqNr26U5ckjcicnYkkuRg4Gtg/yQRwJoO7sR4DrG936l7T7sR6EfCuJDuAXwKnVdVkp/ybGNzp9TgGfSiT/SjvBS5NcipwJ/CquToWSdJwcxYiVbVqSPlj06x7GXDZNMs2AocNqd8DHLM7bZQk7R6fWJck9WaISJJ6M0QkSb0ZIpKk3gwRSVJvhogkqTdDRJLUmyEiSerNEJEk9WaISJJ6M0QkSb0ZIpKk3gwRSVJvhogkqTdDRJLUmyEiSeptRiGS5OqZ1CRJ88tOf9kwyWOBX2PwE7f7AmmL9gGeNsdtkyQ9zO3q53HfCLyFQWBcx/0h8hPg3DlslyRpD7DTy1lV9T+qagnw9qp6elUtaa/nVtVf7mrnSdYmuTvJTZ3afknWJ7m1ve/b6knywSSbktyQ5HmdbVa39W9NsrpTPyLJjW2bDyYJkqSRmVGfSFV9KMlvJ3ltklMmXzPY9AJgxZTaGcDVVbUUuLrNAxwPLG2vNcB5MAgd4Ezg+cCRwJmTwdPWWdPZbupnSZLm0Ew71v8ncA7wO8BvttfyXW1XVV8Ftk8prwQubNMXAid26hfVwDXAk5McCBwHrK+q7VX1Y2A9sKIt26eqvllVBVzU2ZckaQR21ScyaTlwaPuf9e46oKq2AFTVliRPbfWDgLs660202s7qE0PqkqQRmelzIjcB/2IuG8L9nfZd1aP+4B0na5JsTLJx27Ztu9FESVLXTENkf+C7Sa5Ksm7y1fMzt7ZLUbT3u1t9Aji4s94iYPMu6ouG1B+kqs6vquVVtXzhwoU9my1Jmmqml7POmsXPXAesBt7b3q/o1E9PcgmDTvR/bJe7rgLe0+lMPxZ4Z1VtT3JvkqOAbwGnAB+axXZKknZhRiFSVV/ps/MkFwNHM3hYcYLBXVbvBS5NcipwJ/CqtvqVwAnAJuDnwOvbZ29P8m5gQ1vvXVU12Vn/JgZ3gD0O+Hx7SZJGZEYhkuRe7u9v2Bt4NPCzqtpnZ9tV1appFh0zZN0C3jzNftYCa4fUNwKH7awNkqS5M9MzkSd255OcyOCZDUnSPNZrFN+q+gzw4lluiyRpDzPTy1mv6Mw+isFzI7PxzIgkaQ8207uzXtaZ3gHcweAJc0nSPDbTPpHXz3VDJEl7npmOnbUoyeVtRN6tSS5LsmjXW0qSHslm2rH+1wweBnwag/GpPttqkqR5bKYhsrCq/rqqdrTXBYDjh0jSPDfTEPlRkt9PsqC9fh+4Zy4bJkl6+JtpiLwBeDXw/4AtwCtpw5JIkuavmd7i+25gdftRqMlfGzyHQbhIkuapmZ6JPGcyQGAwKCJw+Nw0SZK0p5hpiDyqMxT75JnITM9iJEmPUDMNgvcD30jyaQbDnbwaOHvOWiVJ2iPM9In1i5JsZDDoYoBXVNV357RlkqSHvRlfkmqhYXBIkv5Zr6HgJUkCQ0SStBsMEUlSb4aIJKm3kYdIkmclub7z+kmStyQ5K8kPO/UTOtu8M8mmJLckOa5TX9Fqm5KcMepjkaT5buQPDFbVLcAygCQLgB8ClzMYi+sDVXVOd/0khwInA89mMBT9l5I8sy0+F3gpMAFsSLLOW48laXTG/dT5McAPqurvk0y3zkrgkqq6D7g9ySbgyLZsU1XdBpDkkrauISJJIzLuEDkZuLgzf3qSU4CNwNvaeF0HAdd01ploNYC7ptSfP+xDkqwB1gAccsghu93oI/74ot3ehx5ZrvvzU8bdBGksxtaxnmRv4OXAp1rpPOAZDC51bWEw1AoMnpCfqnZSf3Cx6vyqWl5Vyxcu9Le0JGm2jPNM5Hjgb6tqK8DkO0CSjwCfa7MTwMGd7RYBm9v0dHVJ0giM8xbfVXQuZSU5sLPsJOCmNr0OODnJY5IsAZYC1wIbgKVJlrSzmpPbupKkERnLmUiSX2NwV9UbO+X3JVnG4JLUHZPLqurmJJcy6DDfAby5qn7Z9nM6cBWwAFhbVTeP7CAkSeMJkar6OfCUKbU/2Mn6ZzNk6PmquhK4ctYbKEmaEZ9YlyT1ZohIknozRCRJvRkikqTeDBFJUm+GiCSpN0NEktSbISJJ6s0QkST1ZohIknozRCRJvRkikqTeDBFJUm+GiCSpN0NEktSbISJJ6s0QkST1ZohIknozRCRJvY0tRJLckeTGJNcn2dhq+yVZn+TW9r5vqyfJB5NsSnJDkud19rO6rX9rktXjOh5Jmo/GfSbyb6tqWVUtb/NnAFdX1VLg6jYPcDywtL3WAOfBIHSAM4HnA0cCZ04GjyRp7o07RKZaCVzYpi8ETuzUL6qBa4AnJzkQOA5YX1Xbq+rHwHpgxagbLUnz1ThDpIAvJrkuyZpWO6CqtgC096e2+kHAXZ1tJ1ptuvoDJFmTZGOSjdu2bZvlw5Ck+WuvMX72C6pqc5KnAuuTfH8n62ZIrXZSf2Ch6nzgfIDly5c/aLkkqZ+xnYlU1eb2fjdwOYM+ja3tMhXt/e62+gRwcGfzRcDmndQlSSMwlhBJ8vgkT5ycBo4FbgLWAZN3WK0GrmjT64BT2l1aRwH/2C53XQUcm2Tf1qF+bKtJkkZgXJezDgAuTzLZhk9U1ReSbAAuTXIqcCfwqrb+lcAJwCbg58DrAapqe5J3Axvaeu+qqu2jOwxJmt/GEiJVdRvw3CH1e4BjhtQLePM0+1oLrJ3tNkqSdu3hdouvJGkPYohIknozRCRJvRkikqTeDBFJUm+GiCSpN0NEktSbISJJ6s0QkST1ZohIknozRCRJvRkikqTeDBFJUm+GiCSpN0NEktSbISJJ6s0QkST1ZohIknozRCRJvY08RJIcnOTLSb6X5OYk/7HVz0rywyTXt9cJnW3emWRTkluSHNepr2i1TUnOGPWxSNJ8t9cYPnMH8Laq+tskTwSuS7K+LftAVZ3TXTnJocDJwLOBpwFfSvLMtvhc4KXABLAhybqq+u5IjkKSNPoQqaotwJY2fW+S7wEH7WSTlcAlVXUfcHuSTcCRbdmmqroNIMklbV1DRJJGZKx9IkkWA4cD32ql05PckGRtkn1b7SDgrs5mE602XX3Y56xJsjHJxm3bts3iEUjS/Da2EEnyBOAy4C1V9RPgPOAZwDIGZyrvn1x1yOa1k/qDi1XnV9Xyqlq+cOHC3W67JGlgHH0iJHk0gwD5eFX9DUBVbe0s/wjwuTY7ARzc2XwRsLlNT1eXJI3AOO7OCvAx4HtV9d879QM7q50E3NSm1wEnJ3lMkiXAUuBaYAOwNMmSJHsz6HxfN4pjkCQNjONM5AXAHwA3Jrm+1f4EWJVkGYNLUncAbwSoqpuTXMqgw3wH8Oaq+iVAktOBq4AFwNqqunmUByJJ89047s76GsP7M67cyTZnA2cPqV+5s+0kSXPLJ9YlSb0ZIpKk3gwRSVJvhogkqTdDRJLUmyEiSerNEJEk9WaISJJ6M0QkSb0ZIpKk3gwRSVJvhogkqTdDRJLUmyEiSerNEJEk9WaISJJ6M0QkSb0ZIpKk3gwRSVJvhogkqbc9PkSSrEhyS5JNSc4Yd3skaT7Zo0MkyQLgXOB44FBgVZJDx9sqSZo/9ugQAY4ENlXVbVX1T8AlwMoxt0mS5o29xt2A3XQQcFdnfgJ4/tSVkqwB1rTZnya5ZQRtmy/2B3407kaMW85ZPe4m6MH8bk46M7Oxl18fVtzTQ2TYf5l6UKHqfOD8uW/O/JNkY1UtH3c7pKn8bo7Gnn45awI4uDO/CNg8prZI0ryzp4fIBmBpkiVJ9gZOBtaNuU2SNG/s0ZezqmpHktOBq4AFwNqqunnMzZpvvEyohyu/myOQqgd1IUiSNCN7+uUsSdIYGSKSpN4MkXkqSSV5f2f+7UnO2sU2Ryf57Vluxx1J9p/NfeqRJclJ7fv6r9r84iSv7SxfluSE3di/38HdYIjMX/cBr3iI/3iOBh5SiLShaaTdsQr4GoO7LwEWA6/tLF8G9A4R7R5DZP7aweDulbdOXZDkZUm+leTbSb6U5IAki4HTgLcmuT7JC5NckOSVne1+2t6PTvLlJJ8Abmy1zyS5LsnNbQQBaZeSPAF4AXAq94fIe4EXtu/hO4B3Aa9p869JcmSSb7Tv7zeSPKvta0GSc5LcmOSGJH845bMel+QLSf7DCA9xj7dH3+Kr3XYucEOS902pfw04qqoqyb8H/nNVvS3Jh4GfVtU5AElO3cm+jwQOq6rb2/wbqmp7kscBG5JcVlX3zPLx6JHnROALVfV3SbYneR5wBvD2qvpdgCRbgeVVdXqb3wd4UXsE4CXAe4DfYzD00RLg8LZsv87nPIHB2HsXVdVFIzu6RwBDZB6rqp8kuQj4I+AXnUWLgE8mORDYG7h92Pa7cG0nQAD+KMlJbfpgYClgiGhXVgF/0aYvafP/axfbPAm4MMlSBsMgPbrVXwJ8uKp2AFTV9s42VwDvq6qPz1bD5wsvZ+kvGFwqeHyn9iHgL6vqN4A3Ao+dZtsdtO9QkjAInEk/m5xIcjSDf8C/VVXPBb69k31KACR5CvBi4KNJ7gD+GHgNw8fM63o38OWqOgx4Gfd/18KQsfWarwPHt++xHgJDZJ5rf41dyiBIJj0J+GGb7g5Pey/wxM78HcARbXol9//FN9WTgB9X1c/bHTZH7WazNT+8ksHlpV+vqsVVdTCDs+Jf8cDv4dTvZff7+7pO/YvAaUn2AphyOetPGZwZ/9WsHsE8YIgI4P0Mhs2edBbwqST/lwcOpf1Z4KTJjnXgI8C/SXItgyH4f8ZwXwD2SnIDg78Sr5nl9uuRaRVw+ZTaZQw62Hck+U6StwJfBg6d7FgH3gf81yRfZzAc0qSPAncy6Af8Dg+8wwvgLcBjh/QRaicc9kSS1JtnIpKk3gwRSVJvhogkqTdDRJLUmyEiSerNEJEeZtrYY58bdzukmTBEpIeRyQfhxrW99FAZItIsaL9x8f0kH01yU5KPJ3lJkq8nubWNLDvd6LKvS/KpJJ9l8FR1d7+/2dZ/epLHJ1mbZEOrrdzV9tJc868Wafb8S+BVDEaL3cDgiejfAV4O/AlwCsNHlwX4LeA5baTjowHaD4B9CFhZVXcmeQ/wv6vqDUmeDFyb5EtTtx/FgUqTDBFp9txeVZO/n3IzcHUbTv9GBj+kNN3osgDrpwTAv2bwey/HVtXmVjsWeHmSt7f5xwKHTLO9NBKGiDR77utM/6oz/ysG/9YmR5c9qf3I1//prD913LEtDELicGAyRAL8XlXd0l0xyc7GLZPmlH0i0uhMN7rsMP8A/DvgPZOXt4CrgD+cHK48yeFz0EbpITFEpNGZbnTZoapqK4Pfwzi3nW28m8ElsBuS3NTmpbFyFF9JUm+eiUiSejNEJEm9GSKSpN4MEUlSb4aIJKk3Q0SS1JshIknq7f8Daao+lCL79FYAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QXmYKBcYRWmz"
      },
      "source": [
        "import os,datetime\n",
        "from tensorflow.keras.models import Sequential,Model\n",
        "from tensorflow.keras.layers import Input,Dense,Dropout,LSTM"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hD7xnM2fRWm7",
        "outputId": "55fe835f-0a74-4424-b2f7-27f0b664f594"
      },
      "source": [
        "X_train_3d = np.reshape(X_train.values,(X_train.shape[0],1,X_train.shape[1]))\n",
        "X_test_3d = np.reshape(X_test.values,(X_test.shape[0],1,X_test.shape[1]))\n",
        "print(X_train_3d.shape,\" \",X_test_3d.shape )\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(23231, 1, 120)   (2582, 1, 120)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C0LJS23iRWnD"
      },
      "source": [
        "input_df = Input( shape = (input_shape, ))\n",
        "\n",
        "x = Dense(64, activation = 'relu')(input_df)\n",
        "# x = Dense(32, activation = 'relu', kernel_initializer='glorot_uniform')(x)\n",
        "# x = Dense(16, activation = 'relu', kernel_initializer='glorot_uniform')(x)\n",
        "# x = Dense(4, activation = 'relu', kernel_initializer='glorot_uniform')(x)\n",
        "encoded = Dense(1, activation = 'relu', kernel_initializer='glorot_uniform')(x)\n",
        "# x = Dense(4, activation = 'relu', kernel_initializer='glorot_uniform')(encoded)\n",
        "# x = Dense(16, activation = 'relu', kernel_initializer='glorot_uniform')(x)\n",
        "x = Dense(64, activation = 'relu', kernel_initializer='glorot_uniform')(x)\n",
        "#x = Dense(64, activation = 'relu', kernel_initializer='glorot_uniform')(x)\n",
        "decoded = Dense(input_shape, kernel_initializer='glorot_uniform')(x)\n",
        "\n",
        "autoencoder = Model(input_df, decoded)\n",
        "encoder = Model(input_df, encoded)\n",
        "opt = tf.keras.optimizers.Adam(lr=1e-3)\n",
        "\n",
        "autoencoder.compile(optimizer = opt, loss = 'mean_squared_error')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ek-CycuvRWnK"
      },
      "source": [
        "model = Sequential()\n",
        "model.add(LSTM(64,input_shape=(X_train_3d.shape[1:]),activation='relu',return_sequences=True))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(LSTM(16,activation='relu'))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(1,activation='sigmoid'))\n",
        "\n",
        "opt = tf.keras.optimizers.RMSprop(lr=1e-3)\n",
        "model.compile(loss='binary_crossentropy',\n",
        "              optimizer=opt,\n",
        "              metrics=['accuracy'],)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "1Wn7jqFwRWnL",
        "outputId": "fe4fe896-8db2-4cca-99d2-abb6215735ae"
      },
      "source": [
        "autoencoder.fit(X_train,X_train,epochs=1000,validation_data=(X_test,X_test),verbose=1)#allbacks=[tensorboard_callback])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'pandas.core.frame.DataFrame'>, <class 'NoneType'>\n",
            "Train on 23231 samples, validate on 2582 samples\n",
            "Epoch 1/1000\n",
            "23231/23231 [==============================] - 5s 232us/sample - loss: 0.0241 - val_loss: 0.0054\n",
            "Epoch 2/1000\n",
            "23231/23231 [==============================] - 5s 216us/sample - loss: 0.0034 - val_loss: 0.0024\n",
            "Epoch 3/1000\n",
            "23231/23231 [==============================] - 5s 227us/sample - loss: 0.0017 - val_loss: 0.0014\n",
            "Epoch 4/1000\n",
            "23231/23231 [==============================] - 5s 215us/sample - loss: 0.0012 - val_loss: 0.0011\n",
            "Epoch 5/1000\n",
            "23231/23231 [==============================] - 5s 220us/sample - loss: 9.1738e-04 - val_loss: 9.1654e-04\n",
            "Epoch 6/1000\n",
            "23231/23231 [==============================] - 5s 223us/sample - loss: 7.2850e-04 - val_loss: 7.6126e-04\n",
            "Epoch 7/1000\n",
            "23231/23231 [==============================] - 5s 211us/sample - loss: 6.0797e-04 - val_loss: 6.8202e-04\n",
            "Epoch 8/1000\n",
            "23231/23231 [==============================] - 5s 206us/sample - loss: 5.1686e-04 - val_loss: 5.7259e-04\n",
            "Epoch 9/1000\n",
            "23231/23231 [==============================] - 5s 218us/sample - loss: 4.5758e-04 - val_loss: 5.1607e-04\n",
            "Epoch 10/1000\n",
            "23231/23231 [==============================] - 5s 219us/sample - loss: 4.0475e-04 - val_loss: 4.9747e-04\n",
            "Epoch 11/1000\n",
            "23231/23231 [==============================] - 5s 223us/sample - loss: 3.7414e-04 - val_loss: 4.2673e-04\n",
            "Epoch 12/1000\n",
            "23231/23231 [==============================] - 5s 227us/sample - loss: 3.3836e-04 - val_loss: 4.0012e-04\n",
            "Epoch 13/1000\n",
            "23231/23231 [==============================] - 5s 233us/sample - loss: 3.1431e-04 - val_loss: 3.7528e-04\n",
            "Epoch 14/1000\n",
            "23231/23231 [==============================] - 6s 241us/sample - loss: 3.0301e-04 - val_loss: 3.4853e-04\n",
            "Epoch 15/1000\n",
            "23231/23231 [==============================] - 5s 235us/sample - loss: 2.7743e-04 - val_loss: 3.4411e-04\n",
            "Epoch 16/1000\n",
            "23231/23231 [==============================] - 5s 223us/sample - loss: 2.7561e-04 - val_loss: 3.2843e-04\n",
            "Epoch 17/1000\n",
            "23231/23231 [==============================] - 5s 204us/sample - loss: 2.5325e-04 - val_loss: 3.1823e-04\n",
            "Epoch 18/1000\n",
            "23231/23231 [==============================] - 5s 220us/sample - loss: 2.4263e-04 - val_loss: 3.4826e-04\n",
            "Epoch 19/1000\n",
            "23231/23231 [==============================] - 5s 203us/sample - loss: 2.3997e-04 - val_loss: 2.7893e-04\n",
            "Epoch 20/1000\n",
            "23231/23231 [==============================] - 5s 225us/sample - loss: 2.2167e-04 - val_loss: 3.3711e-04\n",
            "Epoch 21/1000\n",
            "23231/23231 [==============================] - 5s 206us/sample - loss: 2.2273e-04 - val_loss: 2.7801e-04\n",
            "Epoch 22/1000\n",
            "23231/23231 [==============================] - 5s 195us/sample - loss: 2.1896e-04 - val_loss: 2.7312e-04\n",
            "Epoch 23/1000\n",
            "23231/23231 [==============================] - 5s 225us/sample - loss: 2.0808e-04 - val_loss: 3.6965e-04\n",
            "Epoch 24/1000\n",
            "23231/23231 [==============================] - 6s 256us/sample - loss: 2.0532e-04 - val_loss: 3.0015e-04\n",
            "Epoch 25/1000\n",
            "23231/23231 [==============================] - 5s 236us/sample - loss: 2.1202e-04 - val_loss: 2.9930e-04\n",
            "Epoch 26/1000\n",
            "23231/23231 [==============================] - 5s 236us/sample - loss: 1.9663e-04 - val_loss: 2.6530e-04\n",
            "Epoch 27/1000\n",
            "23231/23231 [==============================] - 5s 218us/sample - loss: 2.1121e-04 - val_loss: 3.0123e-04\n",
            "Epoch 28/1000\n",
            "23231/23231 [==============================] - 5s 202us/sample - loss: 1.9135e-04 - val_loss: 2.5207e-04\n",
            "Epoch 29/1000\n",
            "23231/23231 [==============================] - 5s 210us/sample - loss: 1.9671e-04 - val_loss: 3.7047e-04\n",
            "Epoch 30/1000\n",
            "23231/23231 [==============================] - 5s 217us/sample - loss: 1.9015e-04 - val_loss: 2.4755e-04\n",
            "Epoch 31/1000\n",
            "23231/23231 [==============================] - 5s 220us/sample - loss: 1.9096e-04 - val_loss: 2.4178e-04\n",
            "Epoch 32/1000\n",
            "23231/23231 [==============================] - 5s 202us/sample - loss: 1.9846e-04 - val_loss: 2.5197e-04\n",
            "Epoch 33/1000\n",
            "23231/23231 [==============================] - 5s 208us/sample - loss: 1.8122e-04 - val_loss: 2.8246e-04\n",
            "Epoch 34/1000\n",
            "  320/23231 [..............................] - ETA: 8s - loss: 2.1912e-04"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-32-a16ac83c7394>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mautoencoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m#allbacks=[tensorboard_callback])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0mmax_queue_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_queue_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    727\u001b[0m         \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 728\u001b[0;31m         use_multiprocessing=use_multiprocessing)\n\u001b[0m\u001b[1;32m    729\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    730\u001b[0m   def evaluate(self,\n",
            "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, **kwargs)\u001b[0m\n\u001b[1;32m    672\u001b[0m         \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    673\u001b[0m         \u001b[0mvalidation_freq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_freq\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 674\u001b[0;31m         steps_name='steps_per_epoch')\n\u001b[0m\u001b[1;32m    675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    676\u001b[0m   def evaluate(self,\n",
            "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps, validation_freq, mode, validation_in_fit, prepared_feed_values_from_dataset, steps_name, **kwargs)\u001b[0m\n\u001b[1;32m    391\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    392\u001b[0m         \u001b[0;31m# Get outputs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 393\u001b[0;31m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    394\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    395\u001b[0m           \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3738\u001b[0m         \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3739\u001b[0m       \u001b[0mconverted_inputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3740\u001b[0;31m     \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mconverted_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3741\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3742\u001b[0m     \u001b[0;31m# EagerTensor.numpy() will often make a copy to ensure memory safety.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1079\u001b[0m       \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mFor\u001b[0m \u001b[0minvalid\u001b[0m \u001b[0mpositional\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mkeyword\u001b[0m \u001b[0margument\u001b[0m \u001b[0mcombinations\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1080\u001b[0m     \"\"\"\n\u001b[0;32m-> 1081\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1082\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1083\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1119\u001b[0m       raise TypeError(\"Keyword arguments {} unknown. Expected {}.\".format(\n\u001b[1;32m   1120\u001b[0m           list(kwargs.keys()), list(self._arg_keywords)))\n\u001b[0;32m-> 1121\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1122\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1123\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1222\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1223\u001b[0m       flat_outputs = forward_function.call(\n\u001b[0;32m-> 1224\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager)\n\u001b[0m\u001b[1;32m   1225\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1226\u001b[0m       \u001b[0mgradient_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delayed_rewrite_functions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    509\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    510\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"executor_type\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"config_proto\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 511\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    512\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    513\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tensorflow.TFE_Py_Execute(ctx._handle, device_name,\n\u001b[1;32m     60\u001b[0m                                                \u001b[0mop_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m                                                num_outputs)\n\u001b[0m\u001b[1;32m     62\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9D7f5XmdRWnV"
      },
      "source": [
        "1) previously there was an error: the losses were 'nan', because in input data inself there was 'nan', I removed       it and now losses are showing.\n",
        "2) 2nd problem occured when Network was not learning. i.e. at every Epoch the Accuracies were exactly same. I increased the learning rate from 1e-5 to 1e-2. Increasing Learning Rate will definately boost the learning and it works!!!\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CUR1EvnjRWnW"
      },
      "source": [
        "losses = pd.DataFrame(model.history.history)\n",
        "losses[['loss','val_loss','accuracy','val_accuracy']].plot()\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0DN-97XzRWnY"
      },
      "source": [
        "def calc_accuracy(y,y_pred):\n",
        "    count = 0\n",
        "    precount=0\n",
        "    m = int(y.max())\n",
        "    shape = y.shape[0]\n",
        "    for j in range(0,m,1):\n",
        "        for i in range(0,shape-1):\n",
        "            if (y[i]> j and y_pred[i]==1):\n",
        "                count = count + 1\n",
        "            if (y[i]< j and y_pred[i]==0):\n",
        "                count = count + 1\n",
        "        if count > precount:\n",
        "            print(count,\" pre= \",precount)\n",
        "            precount = count\n",
        "        else:\n",
        "            print(count)\n",
        "        count=0\n",
        "    return (precount/shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZyiEkrSyRWnc",
        "outputId": "c2c21bb2-8995-4f44-f6f9-098c9448e966"
      },
      "source": [
        "pred = encoder.predict(X_train)\n",
        "pred"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-0.],\n",
              "       [-0.],\n",
              "       [-0.],\n",
              "       ...,\n",
              "       [-0.],\n",
              "       [-0.],\n",
              "       [-0.]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sv1ESAnRRWni",
        "outputId": "a4c17bff-3464-414f-fbaa-95e342438033"
      },
      "source": [
        "pred = pd.DataFrame(pred,columns=['pred'])\n",
        "y_train= pd.DataFrame(y_train,columns=['y_train'])\n",
        "pred = pred*100\n",
        "pred.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>pred</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>count</td>\n",
              "      <td>23231.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>mean</td>\n",
              "      <td>0.006809</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>std</td>\n",
              "      <td>1.037842</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>min</td>\n",
              "      <td>-0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>25%</td>\n",
              "      <td>-0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>50%</td>\n",
              "      <td>-0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>75%</td>\n",
              "      <td>-0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>max</td>\n",
              "      <td>158.185959</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "               pred\n",
              "count  23231.000000\n",
              "mean       0.006809\n",
              "std        1.037842\n",
              "min       -0.000000\n",
              "25%       -0.000000\n",
              "50%       -0.000000\n",
              "75%       -0.000000\n",
              "max      158.185959"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "azwf2mazRWnm",
        "outputId": "1cfddcc9-e5aa-43c4-9bb3-d096d4995b87"
      },
      "source": [
        "print(calc_accuracy(pred.values,y_train.values))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1  pre=  0\n",
            "16454  pre=  1\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "16454\n",
            "0.7082777323404073\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}