bank-account-fraud-dataset-neurips-2022.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-12-30T21:23:47.145738Z",
     "iopub.status.busy": "2022-12-30T21:23:47.145339Z",
     "iopub.status.idle": "2022-12-30T21:23:47.158413Z",
     "shell.execute_reply": "2022-12-30T21:23:47.157108Z",
     "shell.execute_reply.started": "2022-12-30T21:23:47.145706Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-30T21:23:47.647325Z",
     "iopub.status.busy": "2022-12-30T21:23:47.646950Z",
     "iopub.status.idle": "2022-12-30T21:23:51.164949Z",
     "shell.execute_reply": "2022-12-30T21:23:51.163741Z",
     "shell.execute_reply.started": "2022-12-30T21:23:47.647295Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('C:/Users/matth/OneDrive/Desktop/Education/Base.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-30T21:23:51.167101Z",
     "iopub.status.busy": "2022-12-30T21:23:51.166784Z",
     "iopub.status.idle": "2022-12-30T21:23:51.436763Z",
     "shell.execute_reply": "2022-12-30T21:23:51.435718Z",
     "shell.execute_reply.started": "2022-12-30T21:23:51.167072Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1000000 entries, 0 to 999999\n",
      "Data columns (total 32 columns):\n",
      " #   Column                            Non-Null Count    Dtype  \n",
      "---  ------                            --------------    -----  \n",
      " 0   fraud_bool                        1000000 non-null  int64  \n",
      " 1   income                            1000000 non-null  float64\n",
      " 2   name_email_similarity             1000000 non-null  float64\n",
      " 3   prev_address_months_count         1000000 non-null  int64  \n",
      " 4   current_address_months_count      1000000 non-null  int64  \n",
      " 5   customer_age                      1000000 non-null  int64  \n",
      " 6   days_since_request                1000000 non-null  float64\n",
      " 7   intended_balcon_amount            1000000 non-null  float64\n",
      " 8   payment_type                      1000000 non-null  object \n",
      " 9   zip_count_4w                      1000000 non-null  int64  \n",
      " 10  velocity_6h                       1000000 non-null  float64\n",
      " 11  velocity_24h                      1000000 non-null  float64\n",
      " 12  velocity_4w                       1000000 non-null  float64\n",
      " 13  bank_branch_count_8w              1000000 non-null  int64  \n",
      " 14  date_of_birth_distinct_emails_4w  1000000 non-null  int64  \n",
      " 15  employment_status                 1000000 non-null  object \n",
      " 16  credit_risk_score                 1000000 non-null  int64  \n",
      " 17  email_is_free                     1000000 non-null  int64  \n",
      " 18  housing_status                    1000000 non-null  object \n",
      " 19  phone_home_valid                  1000000 non-null  int64  \n",
      " 20  phone_mobile_valid                1000000 non-null  int64  \n",
      " 21  bank_months_count                 1000000 non-null  int64  \n",
      " 22  has_other_cards                   1000000 non-null  int64  \n",
      " 23  proposed_credit_limit             1000000 non-null  float64\n",
      " 24  foreign_request                   1000000 non-null  int64  \n",
      " 25  source                            1000000 non-null  object \n",
      " 26  session_length_in_minutes         1000000 non-null  float64\n",
      " 27  device_os                         1000000 non-null  object \n",
      " 28  keep_alive_session                1000000 non-null  int64  \n",
      " 29  device_distinct_emails_8w         1000000 non-null  int64  \n",
      " 30  device_fraud_count                1000000 non-null  int64  \n",
      " 31  month                             1000000 non-null  int64  \n",
      "dtypes: float64(9), int64(18), object(5)\n",
      "memory usage: 244.1+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-30T21:23:51.438153Z",
     "iopub.status.busy": "2022-12-30T21:23:51.437863Z",
     "iopub.status.idle": "2022-12-30T21:23:51.466585Z",
     "shell.execute_reply": "2022-12-30T21:23:51.465355Z",
     "shell.execute_reply.started": "2022-12-30T21:23:51.438127Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fraud_bool</th>\n",
       "      <th>income</th>\n",
       "      <th>name_email_similarity</th>\n",
       "      <th>prev_address_months_count</th>\n",
       "      <th>current_address_months_count</th>\n",
       "      <th>customer_age</th>\n",
       "      <th>days_since_request</th>\n",
       "      <th>intended_balcon_amount</th>\n",
       "      <th>payment_type</th>\n",
       "      <th>zip_count_4w</th>\n",
       "      <th>...</th>\n",
       "      <th>has_other_cards</th>\n",
       "      <th>proposed_credit_limit</th>\n",
       "      <th>foreign_request</th>\n",
       "      <th>source</th>\n",
       "      <th>session_length_in_minutes</th>\n",
       "      <th>device_os</th>\n",
       "      <th>keep_alive_session</th>\n",
       "      <th>device_distinct_emails_8w</th>\n",
       "      <th>device_fraud_count</th>\n",
       "      <th>month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.166828</td>\n",
       "      <td>-1</td>\n",
       "      <td>88</td>\n",
       "      <td>50</td>\n",
       "      <td>0.020925</td>\n",
       "      <td>-1.331345</td>\n",
       "      <td>AA</td>\n",
       "      <td>769</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>500.0</td>\n",
       "      <td>0</td>\n",
       "      <td>INTERNET</td>\n",
       "      <td>3.888115</td>\n",
       "      <td>windows</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.296286</td>\n",
       "      <td>-1</td>\n",
       "      <td>144</td>\n",
       "      <td>50</td>\n",
       "      <td>0.005418</td>\n",
       "      <td>-0.816224</td>\n",
       "      <td>AB</td>\n",
       "      <td>366</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>0</td>\n",
       "      <td>INTERNET</td>\n",
       "      <td>31.798819</td>\n",
       "      <td>windows</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.044985</td>\n",
       "      <td>-1</td>\n",
       "      <td>132</td>\n",
       "      <td>40</td>\n",
       "      <td>3.108549</td>\n",
       "      <td>-0.755728</td>\n",
       "      <td>AC</td>\n",
       "      <td>870</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>0</td>\n",
       "      <td>INTERNET</td>\n",
       "      <td>4.728705</td>\n",
       "      <td>other</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.159511</td>\n",
       "      <td>-1</td>\n",
       "      <td>22</td>\n",
       "      <td>50</td>\n",
       "      <td>0.019079</td>\n",
       "      <td>-1.205124</td>\n",
       "      <td>AB</td>\n",
       "      <td>810</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>200.0</td>\n",
       "      <td>0</td>\n",
       "      <td>INTERNET</td>\n",
       "      <td>2.047904</td>\n",
       "      <td>linux</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.596414</td>\n",
       "      <td>-1</td>\n",
       "      <td>218</td>\n",
       "      <td>50</td>\n",
       "      <td>0.004441</td>\n",
       "      <td>-0.773276</td>\n",
       "      <td>AB</td>\n",
       "      <td>890</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>0</td>\n",
       "      <td>INTERNET</td>\n",
       "      <td>3.775225</td>\n",
       "      <td>macintosh</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   fraud_bool  income  name_email_similarity  prev_address_months_count  \\\n",
       "0           1     0.9               0.166828                         -1   \n",
       "1           1     0.9               0.296286                         -1   \n",
       "2           1     0.9               0.044985                         -1   \n",
       "3           1     0.9               0.159511                         -1   \n",
       "4           1     0.9               0.596414                         -1   \n",
       "\n",
       "   current_address_months_count  customer_age  days_since_request  \\\n",
       "0                            88            50            0.020925   \n",
       "1                           144            50            0.005418   \n",
       "2                           132            40            3.108549   \n",
       "3                            22            50            0.019079   \n",
       "4                           218            50            0.004441   \n",
       "\n",
       "   intended_balcon_amount payment_type  zip_count_4w  ...  has_other_cards  \\\n",
       "0               -1.331345           AA           769  ...                0   \n",
       "1               -0.816224           AB           366  ...                0   \n",
       "2               -0.755728           AC           870  ...                0   \n",
       "3               -1.205124           AB           810  ...                1   \n",
       "4               -0.773276           AB           890  ...                0   \n",
       "\n",
       "   proposed_credit_limit  foreign_request    source  \\\n",
       "0                  500.0                0  INTERNET   \n",
       "1                 1500.0                0  INTERNET   \n",
       "2                  200.0                0  INTERNET   \n",
       "3                  200.0                0  INTERNET   \n",
       "4                 1500.0                0  INTERNET   \n",
       "\n",
       "   session_length_in_minutes  device_os  keep_alive_session  \\\n",
       "0                   3.888115    windows                   0   \n",
       "1                  31.798819    windows                   0   \n",
       "2                   4.728705      other                   0   \n",
       "3                   2.047904      linux                   0   \n",
       "4                   3.775225  macintosh                   1   \n",
       "\n",
       "   device_distinct_emails_8w device_fraud_count  month  \n",
       "0                          1                  0      7  \n",
       "1                          1                  0      7  \n",
       "2                          1                  0      7  \n",
       "3                          1                  0      7  \n",
       "4                          1                  0      7  \n",
       "\n",
       "[5 rows x 32 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How many fraudulant examples do we have?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-30T21:23:51.469138Z",
     "iopub.status.busy": "2022-12-30T21:23:51.468803Z",
     "iopub.status.idle": "2022-12-30T21:23:51.476766Z",
     "shell.execute_reply": "2022-12-30T21:23:51.475559Z",
     "shell.execute_reply.started": "2022-12-30T21:23:51.469107Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11029"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['fraud_bool'].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us now encode some dummy variables for some nominal categorical type data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-30T21:23:51.480000Z",
     "iopub.status.busy": "2022-12-30T21:23:51.478671Z",
     "iopub.status.idle": "2022-12-30T21:23:52.143020Z",
     "shell.execute_reply": "2022-12-30T21:23:52.141975Z",
     "shell.execute_reply.started": "2022-12-30T21:23:51.479952Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.get_dummies(data=df, columns=['payment_type','employment_status','housing_status','source'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-30T21:23:52.145118Z",
     "iopub.status.busy": "2022-12-30T21:23:52.144818Z",
     "iopub.status.idle": "2022-12-30T21:23:52.290750Z",
     "shell.execute_reply": "2022-12-30T21:23:52.289487Z",
     "shell.execute_reply.started": "2022-12-30T21:23:52.145090Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1000000 entries, 0 to 999999\n",
      "Data columns (total 49 columns):\n",
      " #   Column                            Non-Null Count    Dtype  \n",
      "---  ------                            --------------    -----  \n",
      " 0   fraud_bool                        1000000 non-null  int64  \n",
      " 1   income                            1000000 non-null  float64\n",
      " 2   name_email_similarity             1000000 non-null  float64\n",
      " 3   prev_address_months_count         1000000 non-null  int64  \n",
      " 4   current_address_months_count      1000000 non-null  int64  \n",
      " 5   customer_age                      1000000 non-null  int64  \n",
      " 6   days_since_request                1000000 non-null  float64\n",
      " 7   intended_balcon_amount            1000000 non-null  float64\n",
      " 8   zip_count_4w                      1000000 non-null  int64  \n",
      " 9   velocity_6h                       1000000 non-null  float64\n",
      " 10  velocity_24h                      1000000 non-null  float64\n",
      " 11  velocity_4w                       1000000 non-null  float64\n",
      " 12  bank_branch_count_8w              1000000 non-null  int64  \n",
      " 13  date_of_birth_distinct_emails_4w  1000000 non-null  int64  \n",
      " 14  credit_risk_score                 1000000 non-null  int64  \n",
      " 15  email_is_free                     1000000 non-null  int64  \n",
      " 16  phone_home_valid                  1000000 non-null  int64  \n",
      " 17  phone_mobile_valid                1000000 non-null  int64  \n",
      " 18  bank_months_count                 1000000 non-null  int64  \n",
      " 19  has_other_cards                   1000000 non-null  int64  \n",
      " 20  proposed_credit_limit             1000000 non-null  float64\n",
      " 21  foreign_request                   1000000 non-null  int64  \n",
      " 22  session_length_in_minutes         1000000 non-null  float64\n",
      " 23  device_os                         1000000 non-null  object \n",
      " 24  keep_alive_session                1000000 non-null  int64  \n",
      " 25  device_distinct_emails_8w         1000000 non-null  int64  \n",
      " 26  device_fraud_count                1000000 non-null  int64  \n",
      " 27  month                             1000000 non-null  int64  \n",
      " 28  payment_type_AA                   1000000 non-null  uint8  \n",
      " 29  payment_type_AB                   1000000 non-null  uint8  \n",
      " 30  payment_type_AC                   1000000 non-null  uint8  \n",
      " 31  payment_type_AD                   1000000 non-null  uint8  \n",
      " 32  payment_type_AE                   1000000 non-null  uint8  \n",
      " 33  employment_status_CA              1000000 non-null  uint8  \n",
      " 34  employment_status_CB              1000000 non-null  uint8  \n",
      " 35  employment_status_CC              1000000 non-null  uint8  \n",
      " 36  employment_status_CD              1000000 non-null  uint8  \n",
      " 37  employment_status_CE              1000000 non-null  uint8  \n",
      " 38  employment_status_CF              1000000 non-null  uint8  \n",
      " 39  employment_status_CG              1000000 non-null  uint8  \n",
      " 40  housing_status_BA                 1000000 non-null  uint8  \n",
      " 41  housing_status_BB                 1000000 non-null  uint8  \n",
      " 42  housing_status_BC                 1000000 non-null  uint8  \n",
      " 43  housing_status_BD                 1000000 non-null  uint8  \n",
      " 44  housing_status_BE                 1000000 non-null  uint8  \n",
      " 45  housing_status_BF                 1000000 non-null  uint8  \n",
      " 46  housing_status_BG                 1000000 non-null  uint8  \n",
      " 47  source_INTERNET                   1000000 non-null  uint8  \n",
      " 48  source_TELEAPP                    1000000 non-null  uint8  \n",
      "dtypes: float64(9), int64(18), object(1), uint8(21)\n",
      "memory usage: 233.7+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning Section\n",
    "We will perform Machine Learning with the target of 'device_os_' type. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-30T21:31:35.651516Z",
     "iopub.status.busy": "2022-12-30T21:31:35.651092Z",
     "iopub.status.idle": "2022-12-30T21:31:36.664180Z",
     "shell.execute_reply": "2022-12-30T21:31:36.662808Z",
     "shell.execute_reply.started": "2022-12-30T21:31:35.651482Z"
    }
   },
   "outputs": [],
   "source": [
    "# Memory with full dataset is ~3TB..\n",
    "minimize_dataset_memory = 1000000\n",
    "\n",
    "# X.shape, y.shape\n",
    "\n",
    "values = ['device_os']\n",
    "predictors =  [i for i in df.columns.values.tolist() if i not in values]\n",
    "target = 'device_os'\n",
    "X = df[predictors].values\n",
    "y = df[target].values\n",
    "X = X[:minimize_dataset_memory, :]\n",
    "y = y[:minimize_dataset_memory]\n",
    "X_train,X_test, y_train, y_test = train_test_split(X,y, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-30T21:17:44.530089Z",
     "iopub.status.busy": "2022-12-30T21:17:44.529628Z",
     "iopub.status.idle": "2022-12-30T21:17:44.680280Z",
     "shell.execute_reply": "2022-12-30T21:17:44.678578Z",
     "shell.execute_reply.started": "2022-12-30T21:17:44.530053Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy for Multiclassification: 0.00\n"
     ]
    }
   ],
   "source": [
    "def softmax(x):\n",
    "    n = np.exp(x) \n",
    "    d = np.sum(n)\n",
    "    return n/d\n",
    "# # making predictions\n",
    "# B = np.random.rand(X1.shape[1],y.shape[1])\n",
    "# pred = X1.dot(B)\n",
    "class MultipleLogRegression:\n",
    "    # softmax\n",
    "    def __init__(self,learning_rate=0.1,n_iters=1000):\n",
    "        self.lr=learning_rate\n",
    "        self.iters=n_iters\n",
    "        self.W=None\n",
    "        \n",
    "    def fit(self,X,y,mu):\n",
    "        features=np.c_[np.ones(X.shape[0]), X]\n",
    "        onehot_encoder = OneHotEncoder(sparse=False)\n",
    "        y_encode=onehot_encoder.fit_transform(y.reshape(-1,1))\n",
    "        self.W=np.zeros((features.shape[1], y_encode.shape[1]))\n",
    "        samples=X.shape[0]\n",
    "        \n",
    "        for i in range(self.iters):\n",
    "            Z=-features@self.W\n",
    "            prob_y=softmax(Z)\n",
    "            error=y_encode-prob_y\n",
    "            dW=1/samples * (features.T @ error) + 2 * mu * self.W\n",
    "            self.W-=self.lr*dW\n",
    "            \n",
    "    def predict(self,X):\n",
    "        # augment the data\n",
    "        features = np.c_[np.ones(X.shape[0]), X]\n",
    "        Z=-features@self.W\n",
    "        y=softmax(Z)\n",
    "        return np.argmax(y,axis=1)\n",
    "clf_ = MultipleLogRegression()\n",
    "clf_.fit(X_train, y_train, 0.01)\n",
    "y_pred = clf_.predict(X_test)\n",
    "accuracy = (y_test == y_pred).mean()\n",
    "print(\"Accuracy for Multiclassification: {:.2f}\".format(accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "Unable to allocate 3.57 TiB for an array with shape (700000, 700000) and data type float64",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[39], line 65\u001b[0m\n\u001b[0;32m     62\u001b[0m         \u001b[39mreturn\u001b[39;00m np\u001b[39m.\u001b[39margmax(P, axis\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m)\n\u001b[0;32m     64\u001b[0m clf__ \u001b[39m=\u001b[39m Multiclass()\n\u001b[1;32m---> 65\u001b[0m clf__\u001b[39m.\u001b[39;49mfit(X_train,y_train)\n\u001b[0;32m     66\u001b[0m y_pred \u001b[39m=\u001b[39m clf__\u001b[39m.\u001b[39mpredict(X_test)\n\u001b[0;32m     67\u001b[0m \u001b[39mprint\u001b[39m(y_pred[:\u001b[39m10\u001b[39m])\n",
      "Cell \u001b[1;32mIn[39], line 49\u001b[0m, in \u001b[0;36mMulticlass.fit\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m     48\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mfit\u001b[39m(\u001b[39mself\u001b[39m, X, y):\n\u001b[1;32m---> 49\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mloss_steps, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mW \u001b[39m=\u001b[39m gradient_descent(X, y)\n",
      "Cell \u001b[1;32mIn[39], line 39\u001b[0m, in \u001b[0;36mgradient_descent\u001b[1;34m(X, Y, max_iter, eta, mu)\u001b[0m\n\u001b[0;32m     37\u001b[0m     step_lst\u001b[39m.\u001b[39mappend(step)\n\u001b[0;32m     38\u001b[0m     W_lst\u001b[39m.\u001b[39mappend(W)\n\u001b[1;32m---> 39\u001b[0m     loss_lst\u001b[39m.\u001b[39mappend(loss(X, Y_onehot, W))\n\u001b[0;32m     41\u001b[0m df \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39mDataFrame({\n\u001b[0;32m     42\u001b[0m     \u001b[39m'\u001b[39m\u001b[39mstep\u001b[39m\u001b[39m'\u001b[39m: step_lst, \n\u001b[0;32m     43\u001b[0m     \u001b[39m'\u001b[39m\u001b[39mloss\u001b[39m\u001b[39m'\u001b[39m: loss_lst\n\u001b[0;32m     44\u001b[0m })\n\u001b[0;32m     45\u001b[0m \u001b[39mreturn\u001b[39;00m df, W\n",
      "Cell \u001b[1;32mIn[39], line 10\u001b[0m, in \u001b[0;36mloss\u001b[1;34m(X, Y, W)\u001b[0m\n\u001b[0;32m      8\u001b[0m Z \u001b[39m=\u001b[39m \u001b[39m-\u001b[39m X \u001b[39m@\u001b[39m W\n\u001b[0;32m      9\u001b[0m N \u001b[39m=\u001b[39m X\u001b[39m.\u001b[39mshape[\u001b[39m0\u001b[39m]\n\u001b[1;32m---> 10\u001b[0m loss \u001b[39m=\u001b[39m \u001b[39m1\u001b[39m\u001b[39m/\u001b[39mN \u001b[39m*\u001b[39m (np\u001b[39m.\u001b[39mtrace(X \u001b[39m@\u001b[39;49m W \u001b[39m@\u001b[39;49m Y\u001b[39m.\u001b[39;49mT) \u001b[39m+\u001b[39m np\u001b[39m.\u001b[39msum(np\u001b[39m.\u001b[39mlog(np\u001b[39m.\u001b[39msum(np\u001b[39m.\u001b[39mexp(Z), axis\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m))))\n\u001b[0;32m     11\u001b[0m \u001b[39mreturn\u001b[39;00m loss\n",
      "\u001b[1;31mMemoryError\u001b[0m: Unable to allocate 3.57 TiB for an array with shape (700000, 700000) and data type float64"
     ]
    }
   ],
   "source": [
    "from scipy.special import softmax\n",
    "\n",
    "onehot_encoder = OneHotEncoder(sparse=False)\n",
    "def loss(X, Y, W):\n",
    "    \"\"\"\n",
    "    Y: onehot encoded\n",
    "    \"\"\"\n",
    "    Z = - X @ W\n",
    "    N = X.shape[0]\n",
    "    loss = 1/N * (np.trace(X @ W @ Y.T) + np.sum(np.log(np.sum(np.exp(Z), axis=1))))\n",
    "    return loss\n",
    "\n",
    "def gradient(X, Y, W, mu):\n",
    "    \"\"\"\n",
    "    Y: onehot encoded \n",
    "    \"\"\"\n",
    "    Z = - X @ W\n",
    "    P = softmax(Z, axis=1)\n",
    "    N = X.shape[0]\n",
    "    gd = 1/N * (X.T @ (Y - P)) + 2 * mu * W\n",
    "    return gd\n",
    "\n",
    "def gradient_descent(X, Y, max_iter=1000, eta=0.01, mu=0.01):\n",
    "    \"\"\"\n",
    "    Very basic gradient descent algorithm with fixed eta and mu\n",
    "    \"\"\"\n",
    "    Y_onehot = onehot_encoder.fit_transform(Y.reshape(-1,1))\n",
    "    W = np.zeros((X.shape[1], Y_onehot.shape[1]))\n",
    "    step = 0\n",
    "    step_lst = [] \n",
    "    loss_lst = []\n",
    "    W_lst = []\n",
    " \n",
    "    while step < max_iter:\n",
    "        step += 1\n",
    "        W -= eta * gradient(X, Y_onehot, W, mu)\n",
    "        step_lst.append(step)\n",
    "        W_lst.append(W)\n",
    "        loss_lst.append(loss(X, Y_onehot, W))\n",
    "\n",
    "    df = pd.DataFrame({\n",
    "        'step': step_lst, \n",
    "        'loss': loss_lst\n",
    "    })\n",
    "    return df, W\n",
    "\n",
    "class Multiclass:\n",
    "    def fit(self, X, y):\n",
    "        self.loss_steps, self.W = gradient_descent(X, y)\n",
    "\n",
    "    def loss_plot(self):\n",
    "        return self.loss_steps.plot(\n",
    "            x='step', \n",
    "            y='loss',\n",
    "            xlabel='step',\n",
    "            ylabel='loss'\n",
    "        )\n",
    "\n",
    "    def predict(self, H):\n",
    "        Z = - H @ self.W\n",
    "        P = softmax(Z, axis=1)\n",
    "        return np.argmax(P, axis=1)\n",
    "\n",
    "clf__ = Multiclass()\n",
    "clf__.fit(X_train,y_train)\n",
    "y_pred = clf__.predict(X_test)\n",
    "print(y_pred[:10])\n",
    "accuracy = (y_test == y_pred).mean()\n",
    "print(\"Accuracy for Multiclassification: {:.2f}\".format(accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply softmax to each row so that the values in each row sum to 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-30T21:32:01.673165Z",
     "iopub.status.busy": "2022-12-30T21:32:01.672733Z",
     "iopub.status.idle": "2022-12-30T21:33:00.091890Z",
     "shell.execute_reply": "2022-12-30T21:33:00.090634Z",
     "shell.execute_reply.started": "2022-12-30T21:32:01.673133Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\matth\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:814: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3831457142857143\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(multi_class='multinomial', solver='lbfgs')\n",
    "clf.fit(X_train, y_train)\n",
    "print(clf.score(X_train, y_train))\n",
    "# from sklearn import linear_model\n",
    "# clf = linear_model.Lasso(alpha=0.1)\n",
    "\n",
    "# try and predict the device??? \n",
    "#  27  device_os_linux                    1000000 non-null  uint8  \n",
    "#  28  device_os_macintosh               1000000 non-null  uint8  \n",
    "#  29  device_os_other                   1000000 non-null  uint8  \n",
    "#  30  device_os_windows                 1000000 non-null  uint8  \n",
    "#  31  device_os_x11                     1000000 non-null  uint8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-30T21:37:50.054970Z",
     "iopub.status.busy": "2022-12-30T21:37:50.054551Z",
     "iopub.status.idle": "2022-12-30T21:37:51.045198Z",
     "shell.execute_reply": "2022-12-30T21:37:51.043964Z",
     "shell.execute_reply.started": "2022-12-30T21:37:50.054935Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[54023     0 36415  9347     0]\n",
      " [ 6993     0  7053  2120     0]\n",
      " [44167     0 48156 10695     0]\n",
      " [32049     0 33057 13771     0]\n",
      " [  993     0   873   288     0]]\n"
     ]
    }
   ],
   "source": [
    "y_pred = clf.predict(X_test)\n",
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix = confusion_matrix(y_test, y_pred, labels=['linux', 'macintosh', 'other', 'windows', 'x11'])\n",
    "print(confusion_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-30T21:38:14.306420Z",
     "iopub.status.busy": "2022-12-30T21:38:14.305565Z",
     "iopub.status.idle": "2022-12-30T21:38:26.685533Z",
     "shell.execute_reply": "2022-12-30T21:38:26.684452Z",
     "shell.execute_reply.started": "2022-12-30T21:38:14.306367Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\matth\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\matth\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       linux       0.39      0.54      0.45     99785\n",
      "   macintosh       0.00      0.00      0.00     16166\n",
      "       other       0.38      0.47      0.42    103018\n",
      "     windows       0.38      0.17      0.24     78877\n",
      "         x11       0.00      0.00      0.00      2154\n",
      "\n",
      "    accuracy                           0.39    300000\n",
      "   macro avg       0.23      0.24      0.22    300000\n",
      "weighted avg       0.36      0.39      0.36    300000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\matth\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that for the prediction macintosh and x11, we see that we never make any predictions meaning that our data must be upsampled before splitting into training and test sets so that we have an equal number of categories within our target variable. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-12-30T21:49:21.627278Z",
     "iopub.status.busy": "2022-12-30T21:49:21.626874Z",
     "iopub.status.idle": "2022-12-30T21:50:30.461812Z",
     "shell.execute_reply": "2022-12-30T21:50:30.460730Z",
     "shell.execute_reply.started": "2022-12-30T21:49:21.627244Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\matth\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\matth\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       linux       0.44      0.54      0.49     99567\n",
      "   macintosh       0.33      0.00      0.00     16075\n",
      "       other       0.45      0.57      0.50    103185\n",
      "     windows       0.43      0.27      0.33     79018\n",
      "         x11       0.00      0.00      0.00      2155\n",
      "\n",
      "    accuracy                           0.45    300000\n",
      "   macro avg       0.33      0.28      0.26    300000\n",
      "weighted avg       0.43      0.45      0.42    300000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\matth\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.utils import shuffle\n",
    "X, y = shuffle(X,y,random_state=42)\n",
    "X_train,X_test, y_train, y_test = train_test_split(X,y, test_size=0.3, random_state=42)\n",
    "clf = LogisticRegression()\n",
    "clf.fit(X_train, y_train)\n",
    "y_pred = clf.predict(X_test)\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "vscode": {
   "interpreter": {
    "hash": "e50035b3dfd539dbd2dca26cbba61024e04488120c7d4591534a01642a5bb0ca"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}