examples/shallow/pca_and_svm.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "ce734de3",
   "metadata": {},
   "source": [
    "### PCA and SVM\n",
    "\n",
    "Previous example: [/examples/shallow/random_forests.ipynb](https://github.com/serhatsoyer/py4ML/blob/main/examples/shallow/random_forests.ipynb)  \n",
    "Next example: [/examples/shallow/grid_search_and_knn.ipynb](https://github.com/serhatsoyer/py4ML/blob/main/examples/shallow/grid_search_and_knn.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "de952ea4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import make_blobs\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "751ce399",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_of_samples = 2 ** 11\n",
    "num_of_real_features = 3\n",
    "num_of_thrash_features = 5\n",
    "num_of_classes = 4\n",
    "common_std = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "81d77e68",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_real      : <class 'numpy.ndarray'>, (2048, 3), float64\n",
      "data_thrash    : <class 'numpy.ndarray'>, (2048, 5), float64\n",
      "data_aug       : <class 'numpy.ndarray'>, (2048, 8), float64\n",
      "gt             : <class 'numpy.ndarray'>, (2048,), int64\n"
     ]
    }
   ],
   "source": [
    "data_real, gt = make_blobs(n_samples=num_of_samples, n_features=num_of_real_features, centers=num_of_classes, cluster_std=common_std)\n",
    "data_thrash = np.random.normal(0, common_std, (num_of_samples, num_of_thrash_features))\n",
    "data_aug = np.hstack((data_real, data_thrash))\n",
    "\n",
    "def print_general_info(input, msg): print(f'{msg:15}: {type(input)}, {input.shape}, {input.dtype}')\n",
    "def print_stats(data, msg): print(f'{msg:15}: Mean: {np.mean(data, axis=0)}, STD: {np.std(data, axis=0)}')\n",
    "\n",
    "print_general_info(data_real, 'data_real')\n",
    "print_general_info(data_thrash, 'data_thrash')\n",
    "print_general_info(data_aug, 'data_aug')\n",
    "print_general_info(gt, 'gt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a0d33bcc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Real:\n",
      "X_train_real   : <class 'numpy.ndarray'>, (307, 3), float64\n",
      "X_test_real    : <class 'numpy.ndarray'>, (1741, 3), float64\n",
      "y_train_real   : <class 'numpy.ndarray'>, (307,), int64\n",
      "y_test_real    : <class 'numpy.ndarray'>, (1741,), int64\n",
      "X_train_real   : Mean: [ 3.40590754 -4.17690549 -0.1857369 ], STD: [7.61220695 5.13348313 6.29651449]\n",
      "\n",
      "Thrash:\n",
      "X_train_thrash : <class 'numpy.ndarray'>, (307, 5), float64\n",
      "X_test_thrash  : <class 'numpy.ndarray'>, (1741, 5), float64\n",
      "y_train_thrash : <class 'numpy.ndarray'>, (307,), int64\n",
      "y_test_thrash  : <class 'numpy.ndarray'>, (1741,), int64\n",
      "X_train_thrash : Mean: [0.18822775 0.2655373  0.14434949 0.21399522 0.06651125], STD: [2.92809975 3.1019975  2.97671555 3.05662955 3.0982358 ]\n",
      "\n",
      "Augmented:\n",
      "X_train_aug    : <class 'numpy.ndarray'>, (307, 8), float64\n",
      "X_test_aug     : <class 'numpy.ndarray'>, (1741, 8), float64\n",
      "y_train_aug    : <class 'numpy.ndarray'>, (307,), int64\n",
      "y_test_aug     : <class 'numpy.ndarray'>, (1741,), int64\n",
      "X_train_aug    : Mean: [ 3.40590754 -4.17690549 -0.1857369   0.18822775  0.2655373   0.14434949\n",
      "  0.21399522  0.06651125], STD: [7.61220695 5.13348313 6.29651449 2.92809975 3.1019975  2.97671555\n",
      " 3.05662955 3.0982358 ]\n"
     ]
    }
   ],
   "source": [
    "print('Real:')\n",
    "X_train_real, X_test_real, y_train_real, y_test_real = train_test_split(data_real, gt, test_size=.85, random_state=1)\n",
    "print_general_info(X_train_real, 'X_train_real')\n",
    "print_general_info(X_test_real, 'X_test_real')\n",
    "print_general_info(y_train_real, 'y_train_real')\n",
    "print_general_info(y_test_real, 'y_test_real')\n",
    "print_stats(X_train_real, 'X_train_real')\n",
    "\n",
    "print('\\nThrash:')\n",
    "X_train_thrash, X_test_thrash, y_train_thrash, y_test_thrash = train_test_split(data_thrash, gt, test_size=.85, random_state=1)\n",
    "print_general_info(X_train_thrash, 'X_train_thrash')\n",
    "print_general_info(X_test_thrash, 'X_test_thrash')\n",
    "print_general_info(y_train_thrash, 'y_train_thrash')\n",
    "print_general_info(y_test_thrash, 'y_test_thrash')\n",
    "print_stats(X_train_thrash, 'X_train_thrash')\n",
    "\n",
    "print('\\nAugmented:')\n",
    "X_train_aug, X_test_aug, y_train_aug, y_test_aug = train_test_split(data_aug, gt, test_size=.85, random_state=1)\n",
    "print_general_info(X_train_aug, 'X_train_aug')\n",
    "print_general_info(X_test_aug, 'X_test_aug')\n",
    "print_general_info(y_train_aug, 'y_train_aug')\n",
    "print_general_info(y_test_aug, 'y_test_aug')\n",
    "print_stats(X_train_aug, 'X_train_aug')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "88ea632b",
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_to4 = PCA(n_components=4)\n",
    "pca_to3 = PCA(n_components=3)\n",
    "pca_to2 = PCA(n_components=2)\n",
    "pca_to1 = PCA(n_components=1)\n",
    "\n",
    "pca_to4.fit(X_train_aug)\n",
    "pca_to3.fit(X_train_aug)\n",
    "pca_to2.fit(X_train_aug)\n",
    "pca_to1.fit(X_train_aug)\n",
    "\n",
    "X_train_reduced_to4 = pca_to4.transform(X_train_aug)\n",
    "X_test_reduced_to4 = pca_to4.transform(X_test_aug)\n",
    "X_train_reduced_to3 = pca_to3.transform(X_train_aug)\n",
    "X_test_reduced_to3 = pca_to3.transform(X_test_aug)\n",
    "X_train_reduced_to2 = pca_to2.transform(X_train_aug)\n",
    "X_test_reduced_to2 = pca_to2.transform(X_test_aug)\n",
    "X_train_reduced_to1 = pca_to1.transform(X_train_aug)\n",
    "X_test_reduced_to1 = pca_to1.transform(X_test_aug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fb374b78",
   "metadata": {},
   "outputs": [],
   "source": [
    "def disp_results(func, gt, pred, msg): print(f'{func.__name__} of {msg}:\\n{func(gt, pred)}')\n",
    "\n",
    "def analyse(X_train, X_test, y_train, y_test):\n",
    "    model = SVC()\n",
    "    model.fit(X_train, y_train)\n",
    "    y_train_pred = model.predict(X_train)\n",
    "    y_test_pred = model.predict(X_test)\n",
    "    disp_results(classification_report, y_train, y_train_pred, 'training data')\n",
    "    disp_results(confusion_matrix, y_train, y_train_pred, 'training data')\n",
    "    print()\n",
    "    disp_results(classification_report, y_test, y_test_pred, 'test data')\n",
    "    disp_results(confusion_matrix, y_test, y_test_pred, 'test data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4e19e43e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Real:\n",
      "classification_report of training data:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00        73\n",
      "           1       0.66      0.73      0.69        84\n",
      "           2       0.72      0.68      0.70        78\n",
      "           3       0.91      0.86      0.89        72\n",
      "\n",
      "    accuracy                           0.81       307\n",
      "   macro avg       0.82      0.82      0.82       307\n",
      "weighted avg       0.82      0.81      0.81       307\n",
      "\n",
      "confusion_matrix of training data:\n",
      "[[73  0  0  0]\n",
      " [ 0 61 19  4]\n",
      " [ 0 23 53  2]\n",
      " [ 0  8  2 62]]\n",
      "\n",
      "classification_report of test data:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00       439\n",
      "           1       0.75      0.72      0.73       428\n",
      "           2       0.76      0.83      0.79       434\n",
      "           3       0.90      0.87      0.89       440\n",
      "\n",
      "    accuracy                           0.85      1741\n",
      "   macro avg       0.85      0.85      0.85      1741\n",
      "weighted avg       0.86      0.85      0.85      1741\n",
      "\n",
      "confusion_matrix of test data:\n",
      "[[439   0   0   0]\n",
      " [  0 307  94  27]\n",
      " [  0  61 359  14]\n",
      " [  0  40  18 382]]\n"
     ]
    }
   ],
   "source": [
    "print('Real:'); analyse(X_train_real, X_test_real, y_train_real, y_test_real)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bb32db4d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thrash:\n",
      "classification_report of training data:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.49      0.27      0.35        73\n",
      "           1       0.45      0.68      0.54        84\n",
      "           2       0.55      0.59      0.57        78\n",
      "           3       0.53      0.42      0.47        72\n",
      "\n",
      "    accuracy                           0.50       307\n",
      "   macro avg       0.51      0.49      0.48       307\n",
      "weighted avg       0.50      0.50      0.49       307\n",
      "\n",
      "confusion_matrix of training data:\n",
      "[[20 27 14 12]\n",
      " [ 6 57 13  8]\n",
      " [ 5 20 46  7]\n",
      " [10 22 10 30]]\n",
      "\n",
      "classification_report of test data:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.20      0.11      0.14       439\n",
      "           1       0.23      0.38      0.29       428\n",
      "           2       0.25      0.32      0.28       434\n",
      "           3       0.28      0.16      0.20       440\n",
      "\n",
      "    accuracy                           0.24      1741\n",
      "   macro avg       0.24      0.24      0.23      1741\n",
      "weighted avg       0.24      0.24      0.23      1741\n",
      "\n",
      "confusion_matrix of test data:\n",
      "[[ 47 185 145  62]\n",
      " [ 73 161 138  56]\n",
      " [ 67 163 141  63]\n",
      " [ 47 183 141  69]]\n"
     ]
    }
   ],
   "source": [
    "print('Thrash:'); analyse(X_train_thrash, X_test_thrash, y_train_thrash, y_test_thrash)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "de99b19b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Augmented:\n",
      "classification_report of training data:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00        73\n",
      "           1       0.75      0.79      0.77        84\n",
      "           2       0.77      0.74      0.76        78\n",
      "           3       0.92      0.90      0.91        72\n",
      "\n",
      "    accuracy                           0.85       307\n",
      "   macro avg       0.86      0.86      0.86       307\n",
      "weighted avg       0.85      0.85      0.85       307\n",
      "\n",
      "confusion_matrix of training data:\n",
      "[[73  0  0  0]\n",
      " [ 0 66 14  4]\n",
      " [ 0 18 58  2]\n",
      " [ 0  4  3 65]]\n",
      "\n",
      "classification_report of test data:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00       439\n",
      "           1       0.75      0.73      0.74       428\n",
      "           2       0.77      0.82      0.79       434\n",
      "           3       0.91      0.85      0.88       440\n",
      "\n",
      "    accuracy                           0.85      1741\n",
      "   macro avg       0.85      0.85      0.85      1741\n",
      "weighted avg       0.86      0.85      0.85      1741\n",
      "\n",
      "confusion_matrix of test data:\n",
      "[[439   0   0   0]\n",
      " [  0 314  89  25]\n",
      " [  0  63 357  14]\n",
      " [  0  44  20 376]]\n"
     ]
    }
   ],
   "source": [
    "print('Augmented:'); analyse(X_train_aug, X_test_aug, y_train_aug, y_test_aug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "51ce621c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reduced to 4:\n",
      "classification_report of training data:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00        73\n",
      "           1       0.68      0.75      0.71        84\n",
      "           2       0.74      0.69      0.72        78\n",
      "           3       0.93      0.88      0.90        72\n",
      "\n",
      "    accuracy                           0.82       307\n",
      "   macro avg       0.84      0.83      0.83       307\n",
      "weighted avg       0.83      0.82      0.83       307\n",
      "\n",
      "confusion_matrix of training data:\n",
      "[[73  0  0  0]\n",
      " [ 0 63 17  4]\n",
      " [ 0 23 54  1]\n",
      " [ 0  7  2 63]]\n",
      "\n",
      "classification_report of test data:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00       439\n",
      "           1       0.72      0.71      0.71       428\n",
      "           2       0.75      0.80      0.78       434\n",
      "           3       0.91      0.87      0.89       440\n",
      "\n",
      "    accuracy                           0.85      1741\n",
      "   macro avg       0.85      0.85      0.85      1741\n",
      "weighted avg       0.85      0.85      0.85      1741\n",
      "\n",
      "confusion_matrix of test data:\n",
      "[[439   0   0   0]\n",
      " [  0 302 100  26]\n",
      " [  0  74 349  11]\n",
      " [  0  41  16 383]]\n"
     ]
    }
   ],
   "source": [
    "print('Reduced to 4:'); analyse(X_train_reduced_to4, X_test_reduced_to4, y_train_aug, y_test_aug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5c3062f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reduced to 3:\n",
      "classification_report of training data:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00        73\n",
      "           1       0.65      0.75      0.70        84\n",
      "           2       0.72      0.64      0.68        78\n",
      "           3       0.91      0.86      0.89        72\n",
      "\n",
      "    accuracy                           0.81       307\n",
      "   macro avg       0.82      0.81      0.82       307\n",
      "weighted avg       0.81      0.81      0.81       307\n",
      "\n",
      "confusion_matrix of training data:\n",
      "[[73  0  0  0]\n",
      " [ 0 63 17  4]\n",
      " [ 0 26 50  2]\n",
      " [ 0  8  2 62]]\n",
      "\n",
      "classification_report of test data:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00       439\n",
      "           1       0.72      0.70      0.71       428\n",
      "           2       0.75      0.80      0.77       434\n",
      "           3       0.90      0.86      0.88       440\n",
      "\n",
      "    accuracy                           0.84      1741\n",
      "   macro avg       0.84      0.84      0.84      1741\n",
      "weighted avg       0.84      0.84      0.84      1741\n",
      "\n",
      "confusion_matrix of test data:\n",
      "[[439   0   0   0]\n",
      " [  0 299 100  29]\n",
      " [  0  75 348  11]\n",
      " [  0  44  17 379]]\n"
     ]
    }
   ],
   "source": [
    "print('Reduced to 3:'); analyse(X_train_reduced_to3, X_test_reduced_to3, y_train_aug, y_test_aug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6e8e7bfc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reduced to 2:\n",
      "classification_report of training data:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00        73\n",
      "           1       0.66      0.68      0.67        84\n",
      "           2       0.70      0.69      0.70        78\n",
      "           3       0.90      0.88      0.89        72\n",
      "\n",
      "    accuracy                           0.80       307\n",
      "   macro avg       0.81      0.81      0.81       307\n",
      "weighted avg       0.81      0.80      0.81       307\n",
      "\n",
      "confusion_matrix of training data:\n",
      "[[73  0  0  0]\n",
      " [ 0 57 21  6]\n",
      " [ 0 23 54  1]\n",
      " [ 0  7  2 63]]\n",
      "\n",
      "classification_report of test data:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00       439\n",
      "           1       0.70      0.72      0.71       428\n",
      "           2       0.74      0.79      0.77       434\n",
      "           3       0.91      0.83      0.87       440\n",
      "\n",
      "    accuracy                           0.84      1741\n",
      "   macro avg       0.84      0.83      0.84      1741\n",
      "weighted avg       0.84      0.84      0.84      1741\n",
      "\n",
      "confusion_matrix of test data:\n",
      "[[439   0   0   0]\n",
      " [  0 308  93  27]\n",
      " [  0  81 344   9]\n",
      " [  0  51  25 364]]\n"
     ]
    }
   ],
   "source": [
    "print('Reduced to 2:'); analyse(X_train_reduced_to2, X_test_reduced_to2, y_train_aug, y_test_aug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ab9cbc63",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reduced to 1:\n",
      "classification_report of training data:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00        73\n",
      "           1       0.66      0.75      0.70        84\n",
      "           2       0.50      0.36      0.42        78\n",
      "           3       0.58      0.67      0.62        72\n",
      "\n",
      "    accuracy                           0.69       307\n",
      "   macro avg       0.69      0.69      0.69       307\n",
      "weighted avg       0.68      0.69      0.68       307\n",
      "\n",
      "confusion_matrix of training data:\n",
      "[[73  0  0  0]\n",
      " [ 0 63  9 12]\n",
      " [ 0 27 28 23]\n",
      " [ 0  5 19 48]]\n",
      "\n",
      "classification_report of test data:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00       439\n",
      "           1       0.63      0.65      0.64       428\n",
      "           2       0.37      0.33      0.35       434\n",
      "           3       0.56      0.59      0.57       440\n",
      "\n",
      "    accuracy                           0.65      1741\n",
      "   macro avg       0.64      0.64      0.64      1741\n",
      "weighted avg       0.64      0.65      0.64      1741\n",
      "\n",
      "confusion_matrix of test data:\n",
      "[[439   0   0   0]\n",
      " [  0 279 109  40]\n",
      " [  0 121 145 168]\n",
      " [  1  45 134 260]]\n"
     ]
    }
   ],
   "source": [
    "print('Reduced to 1:'); analyse(X_train_reduced_to1, X_test_reduced_to1, y_train_aug, y_test_aug)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "c484cb68",
   "metadata": {},
   "source": [
    "Previous example: [/examples/shallow/random_forests.ipynb](https://github.com/serhatsoyer/py4ML/blob/main/examples/shallow/random_forests.ipynb)  \n",
    "Next example: [/examples/shallow/grid_search_and_knn.ipynb](https://github.com/serhatsoyer/py4ML/blob/main/examples/shallow/grid_search_and_knn.ipynb)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.6 ('study')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8 (main, Nov 24 2022, 08:08:27) [Clang 14.0.6 ]"
  },
  "vscode": {
   "interpreter": {
    "hash": "29d06b973d1ddb34db3279b24f9b5152402e688db937648b736e855ec4de60c0"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}