speech-recognition-best-model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorFlow Speech Recognition Challenge Code\n",
    "### Table of Contents:\n",
    "1. [Imports](#imports)\n",
    "2. [Read data](#readdata)\n",
    "3. [Clean/augment wav files](#clean)\n",
    "4. [Create spectrograms](#specgrams)\n",
    "5. [Set class weights](#weights)\n",
    "6. [Train CNN](#cnn)\n",
    "7. [Predict on test files](#predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"imports\"></a>\n",
    "## 1. Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import glob\n",
    "import scipy\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import os\n",
    "import librosa\n",
    "import IPython.display as ipd\n",
    "import librosa.display\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"readdata\"></a>\n",
    "## 2. Read pickled DataFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "POSSIBLE_LABELS = 'yes no up down left right on off stop go unknown silence'.split()\n",
    "id2name = {i: name for i, name in enumerate(POSSIBLE_LABELS)}\n",
    "name2id = {name: i for i, name in id2name.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_pickle('train_df_dec10.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_df = pd.read_pickle('valid_df_dec10.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>label_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>wav_file</th>\n",
       "      <th>sound</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>left</td>\n",
       "      <td>4</td>\n",
       "      <td>cb8f8307</td>\n",
       "      <td>train/audio/left/cb8f8307_nohash_1.wav</td>\n",
       "      <td>[-7, 21, -10, 6, -13, 7, 15, -23, 14, -12, 29,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label label_id   user_id                                wav_file  \\\n",
       "0  left        4  cb8f8307  train/audio/left/cb8f8307_nohash_1.wav   \n",
       "\n",
       "                                               sound  \n",
       "0  [-7, 21, -10, 6, -13, 7, 15, -23, 14, -12, 29,...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>label_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>wav_file</th>\n",
       "      <th>sound</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>65122</th>\n",
       "      <td>silence</td>\n",
       "      <td>11</td>\n",
       "      <td>running</td>\n",
       "      <td>train/audio/_background_noise_/running_tap.wav</td>\n",
       "      <td>[-4884, -1366, 1058, 2169, -2183, 1213, 2858, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         label label_id  user_id  \\\n",
       "65122  silence       11  running   \n",
       "\n",
       "                                             wav_file  \\\n",
       "65122  train/audio/_background_noise_/running_tap.wav   \n",
       "\n",
       "                                                   sound  \n",
       "65122  [-4884, -1366, 1058, 2169, -2183, 1213, 2858, ...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.tail(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>label_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>wav_file</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>left</td>\n",
       "      <td>4</td>\n",
       "      <td>471a0925</td>\n",
       "      <td>train/audio/left/471a0925_nohash_4.wav</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label  label_id   user_id                                wav_file\n",
       "0  left         4  471a0925  train/audio/left/471a0925_nohash_4.wav"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_df.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"clean\"></a>\n",
    "## 3. Clean/augment wav files\n",
    "\n",
    "### Make sure all wavs are exactly 1 sec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resize_wavs(x):\n",
    "    length = len(x)\n",
    "    # if shorter than 16k, pad with zeros\n",
    "    if length < 16000:\n",
    "        diff = 16000-length\n",
    "        ind = np.random.randint(0, diff)\n",
    "        x = np.pad(x, (ind, 16000-(ind+length)), \"constant\")\n",
    "    # if longer than 16k, chop off end\n",
    "    elif length > 16000:\n",
    "        x = x[:16000]\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df[\"sound2\"] = train_df.sound.apply(resize_wavs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>label_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>wav_file</th>\n",
       "      <th>sound</th>\n",
       "      <th>sound2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>left</td>\n",
       "      <td>4</td>\n",
       "      <td>cb8f8307</td>\n",
       "      <td>train/audio/left/cb8f8307_nohash_1.wav</td>\n",
       "      <td>[-7, 21, -10, 6, -13, 7, 15, -23, 14, -12, 29,...</td>\n",
       "      <td>[-7, 21, -10, 6, -13, 7, 15, -23, 14, -12, 29,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label label_id   user_id                                wav_file  \\\n",
       "0  left        4  cb8f8307  train/audio/left/cb8f8307_nohash_1.wav   \n",
       "\n",
       "                                               sound  \\\n",
       "0  [-7, 21, -10, 6, -13, 7, 15, -23, 14, -12, 29,...   \n",
       "\n",
       "                                              sound2  \n",
       "0  [-7, 21, -10, 6, -13, 7, 15, -23, 14, -12, 29,...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Waves/labels to lists, separate silence files (for now)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "sound2 = list(train_df.sound2)\n",
    "label_id = list(train_df.label_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "silences = list(train_df[train_df.label_id == 11].sound2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7200"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(silences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Double non-silence training data by smushing, stretching, or adding noise to clips"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "sound3 = sound2\n",
    "label_id2 = label_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# resamples down to 10k, pads shortened clip with noise to return to 16k\n",
    "def shrink(x):\n",
    "    x = scipy.signal.resample(x, 10000)\n",
    "    ind = np.random.randint(0, 6000)\n",
    "    x = np.pad(x, (ind, 16000-(ind+10000)), \"constant\")\n",
    "    sil_ind = np.random.randint(0, len(silences))\n",
    "    new_x = 0.8*x + 0.2*silences[sil_ind]\n",
    "    return new_x\n",
    "\n",
    "# resamples up to 22k, takes middle 16k\n",
    "def stretch(x):\n",
    "    x = scipy.signal.resample(x, 22000)\n",
    "    return x[3000:19000]\n",
    "\n",
    "# layers in background noise from silence files\n",
    "def add_noise(x):\n",
    "    sil_ind = np.random.randint(0, len(silences))\n",
    "    new_x = 0.7*x + 0.4*silences[sil_ind]\n",
    "    return new_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# applies augmentation functions\n",
    "for i in range(57923):\n",
    "    if i % 3 == 0:\n",
    "        new_x = shrink(sound2[i])\n",
    "        sound3.append(new_x)\n",
    "        label_id2.append(label_id[i])\n",
    "    elif i % 3 == 1:\n",
    "        new_x = stretch(sound2[i])\n",
    "        sound3.append(new_x)\n",
    "        label_id2.append(label_id[i])\n",
    "    elif i % 3 == 2:\n",
    "        new_x = add_noise(sound2[i])\n",
    "        sound3.append(new_x)\n",
    "        label_id2.append(label_id[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle.dump(sound3, open('sound3.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle.dump(label_id2, open(\"label_id2.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"specgrams\"></a>\n",
    "## 4. Create spectrograms with Librosa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.io import wavfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def wav2spec_val(X, max_len = 16000, n_mels = 128):\n",
    "    \"\"\"Turns audio into a spectrogram, used on validation data\"\"\"\n",
    "    x_spec = np.zeros((len(X), n_mels, 32, 1))\n",
    "    for i, fn in enumerate(X):\n",
    "        _, wave = scipy.io.wavfile.read(fn)\n",
    "        wave = wave[:max_len]\n",
    "        wave = np.pad(wave, (0, max_len-wave.shape[0]), 'constant')\n",
    "        # scales data from -1 to 1\n",
    "        wave = wave / np.max([np.max(abs(wave)), 0.00001])\n",
    "        S = librosa.feature.melspectrogram(wave, sr=16000, n_mels=n_mels)\n",
    "        log_S = librosa.power_to_db(S, ref=np.max)\n",
    "        x_spec[i,:,:,0] = log_S\n",
    "    return x_spec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def wav2spec_train(X, max_len = 16000, n_mels = 128):\n",
    "    \"\"\"Turns audio into a spectrogram, used on training data\"\"\"\n",
    "    x_spec = np.zeros((len(X), n_mels, 32, 1))\n",
    "    for i, wave in enumerate(X):\n",
    "        wave = wave[:max_len]\n",
    "        wave = np.pad(wave, (0, max_len-wave.shape[0]), 'constant')\n",
    "        # scales data from -1 to 1\n",
    "        wave = wave / np.max([np.max(abs(wave)), 0.00001])\n",
    "        S = librosa.feature.melspectrogram(wave, sr=16000, n_mels=n_mels)\n",
    "        log_S = librosa.power_to_db(S, ref=np.max)\n",
    "        x_spec[i,:,:,0] = log_S\n",
    "    return x_spec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_vec = wav2spec_train(sound3)\n",
    "x_val_vec = wav2spec_val(valid_df.wav_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(123046, 128, 32, 1)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_vec.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_vec = np.array(label_id2)\n",
    "y_val_vec = valid_df.label_id.as_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6798,)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_val_vec.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f0fbd5e8d68>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiIAAADFCAYAAABpapk5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJztnXuw3Vd137/rvF/3raurq4ctv4AQ\nHGOiOHFJiYNLawiJ2xmPJ6SlhpJxOpOkpEkmQDodSKd0nOZB6LQh4waKmSHYDjjF09AkLoEhZMD4\ngY3BNn7IL8mSrqSr+zjv1+4f57ioRtK9Z3+k+zvC6zvjsXR119m/8/vt395rr/Vd32UhBDkcDofD\n4XAkgVTSF+BwOBwOh+OVC3dEHA6Hw+FwJAZ3RBwOh8PhcCQGd0QcDofD4XAkBndEHA6Hw+FwJAZ3\nRBwOh8PhcCQGd0QcDofD4XAkBndEHA6Hw+FwJAZ3RBwOh8PhcCSGTNIXIEnpcjlkp2ej7a0XP7b1\n420HH8DMA7Q3IIyLx4b3DtknLAgc0smOnyTI+0ZB73ugRy/wztD7Rt+3Plztyfh4nU0QSa7RZ2P8\nfhaMDd637vKyerXapq5+LByR7PSsLvylX4+2z9Tix86tsVnSz7JZ0sshc6W68bZkgkpSps7uXboJ\nbNtsbLo4NGfYjma9+AvACxvcjPMr8RdAHYnWFLv4bpGNT94Zct8kKdNA5mrOsrWKvO/pFhoarXMS\nm/PUgUu3mT3dI+o74597ezL+mR/8o49s+nfHwhFJdaTCUrx9ewrYTrCXkzoyRl+wTPz1E0dAGjw3\nBHDrQ4o9tz7cEFMd+NzB6Zguyh045zvleFu6KNP7nqIHh3y8LT20hCb77pUXWViil4u/fur80kNT\ntxR/7XTO9Wn0Ft673Eq8bboF7tsI69RYOCIKUroV/7QyjfibRcOluSqbZR3wgkhSl7ygNK1E0xNg\nfPpy04WNfndyQkyBaIrEHcgkw+zY+YUgz50eOvBmDGP8xJmgcybTYHOeXDtxwIajI2scTcrEj9/v\nkdPi5n91PBwRYy9ZFjgD1BHpFOHJnIb9gAPXhy8YfUFJiiELUzPZGrPv5aEDCVIE2Tobm0bCyKJO\noimSlIIRFfK+SFJ+hZwQ4WYKnd/GPHxfgTOBnxtMT5C1hjpBfXjtnTJ8buDgkl3fmr11PBwRSSEd\nf7NRVACCTjJ6ysmfiJ8o+RV2TKHRHLKw0VAvTcnhMDt48yg/hoI4z9Txx1E4euuAfRceWlLw3iUZ\nCaMREfrcyfueqbOxM/DQgyctMCdzZhTnb8Pl3Mw+YWZLZvbtk372e2b2uJl9y8z+wsymT/q3D5jZ\nU2b2XTP7J6NevMPhcDgcjlcONnO2+aSk/yrpUyf97B5JHwghdM3sdyV9QNL7zOy1kn5e0g9L2inp\n/5jZq0IIG/ryJHxET6cEuIQVnnLIKYvmrCnHBHFEEiSvSSyiQdEtwOoHSHpMkiNCn3vSZd8ElGBN\niLYS4ypgUj6ImEvs3nVLaGgFg2sNfO6kWgtFX88mRySE8BUz2/uyn/3NSX/9uqQbhn++XtLtIYSW\npGfM7ClJV0n62hkHMVb9QR4UzV1SrgEFcUToop5kqDcLc/3nta4BTEvRqhuymRtkGXdgtRQFItVD\nbk6PEqzBGisxrkQOcA0kzkcjnC66R9C1pgdLzrsFMDa4b6Psy2fjXPevJN0x/PMuDRyTl3Bg+LPv\ng5ndLOlmScpWZpQCnhc53bYno00HgN4unqTglJPqJBsVICR+WoJKnTBqn63G21L9lg494YHTKc63\nw82cksPJc6eOBOWjYQ2Xmfjn3ppOrmJHEoq+Yq0neGCj9mStJNGgUd419HjN7N9J6kr69Ki2IYRb\nQwj7Qgj7MgVIpXc4HA6Hw3FeIvp8YGbvkvR2SdeGEF46oh2UtOekX9s9/NmZkWIhIHLSoCqVNPdJ\nQ9UBRIN6CXcaIvwYekKi4U4qu6xKvCkNU+PTJbCnuX4KyskiERGq6krF4Oj4qIIiYXn7JPlofTjl\n0+vQHuxRPZDWGQVRjoiZXSfptyT9VAjh5GDr3ZL+zMz+UAOy6mWSvrHhBwamXpdbT44ngYWxYN5W\npAQ24eJtIllN9SACfLkxQRqVgbKhKdLkucHSY6rfgnlRYFHPrbGx6bXjElrkgLKxiTqoBMXY4GGT\nll13YcKAEHXJfR/le2+4FZnZZyRdI2mbmR2Q9EENqmTyku6xAUfi6yGEfx1C+I6Z3SnpUQ1SNr+8\nmYqZkJbaIIdIyET05SRiahJfWJEeBSSQ0ZM1YZNTkjAVoktDcSpSuZKHizKdcwR0zlB+DG6aR24d\n9V0hUZdu5qjBJnzu1IElomCUo0HXKuL4S0y4knAQR3lXNlM1845T/PjjZ/j9D0v68OYvYXDKKC4l\nU2LUhYtyGk7SkIaqfWAzx+kFCCpVTpCBEZU+jGQRgjVVesyvMu+blA/T+0Y38x50RJCoF20nQfsb\nJVgplnyvmXhb6vjTOUvpA2lw6LEukIcYYb4lzBJwOBwOh8PxSsZYSLyHFAudkRNaGp4ycFQB2pPT\ncXsyOZ6DxMLklLBJ+35QbYEkhehoiB+VM9LTISWbJkiWDZD4R0uPMU8DdBqn/Y2SnDftCTZ2CgoQ\nUpIyee5kzo0SBRsbR4QQcrKEBAY30+Zccm3FJamwTLoWw1Av7SYKNjTqCOB8PXxzCNeBbiiUxZ8F\n105TM1iLA75vhC9AtJIk7kDiPj9E96dC9ZbYvSPrZNJrDVYTTgijpGbGwhEZVM3Em5NJitVBE5aM\nJpEBUqkkSQEK/ZCoQAZ6kDRfjqtmEuo2LUmFFbYjNafjPSEaiWpPsWwyJk0C5x03Sky44V8XRK2p\nKBjdzUnlCo16UxE/fGgC8wY5ryNct3NEHA6Hw+FwJIaxiIiEtNQBeThy0kiazU1BIjJYk4FKToN7\n34fy9DQaRPo3SEwoiPIc+hl2tG5PESY9DNFTXhKtugHvTOKaRXCtywHtHcxvwVIBCY6dYKUWBbpv\nI/zuWDgiqa6UPx5v35qJt+0DW4n3vkjhFAGzJ6Dh1gCunZLfcM4abogkJUj5KZikDEAdgaRF+BBH\nhPYMoX2xEuxrRdMLuF8L4LPRAxfuegw5KkTQLJBrP9/Iqv2M1JoD9lmwK8CXM9OkJ3M2PiH59grJ\nHi+JhgoR6ZFYS3OJLy7tGVDbD+dMup2cRDzNl1NHplNhc741HW+bBQrQ0lloawDfd3ToSdgRsR6I\nZOWgeGE92chzt5wMkXGru+9ymNQDD7tbifcmUi32dpOUksQ3FfKCp+CG1C1BVVmwMGZqsPoCplbw\nyZ4Yw1AvedckIWZZl5IOaQUDRK8I7l2C6QVJCpnkmPVpeGBrzzKCdaoZf/PpOknfN0xSBkgTJ8oF\nzRwOh8PhcJwPGIuISEhL3SmQIwHOMg3Rd8kJSZKA9LAkdLSmJCgcLgVRDRpupKkVKs2P7j0kfNJ7\nF7LxF2/0WE8bt5XgyTofb989ziYdLjkvsA8g6Y1eBd73EhMt6qfJVsdCElS7pl+k/IF4+043PlYR\nRqBMjIUjop6UXgehM+KINJIV2qE56xSoHqHhUlxxBEKWtKMlzdtSR4Z8934ObihU0QxsSDQGSzZD\nSQowFdsH9pQiTN51SQqw2ioAZ8IaML9wlL1w5NyA0hOC6TxJAs7v4AJAc9GV+Oc2yrs6Fo6IBSnd\nAiW4YJZRFj5dHHIrVKo83jZTQ0Pz9tSAhEWdKKoKm63SqES8fWeCOSJ0zhJeFSKWixMHU33miGTB\ngYl+d0oyzlThyX4t/rtTTldnKrlO4bhCDj43W2JsVbJHoLVihPs2Fo6ITOrn4582IT0a7TkCuhNK\nPCJDTubt6WT7rVBnggCnJ+CbQwhsNCqApfnBu0pKCSXWDVSSQLNqjOISJGyCih1Jas3DkzXZkzLJ\n0hFJuT+OIKZg1Q2N2ielKjvC13ayqsPhcDgcjsQwHhGRAMtYiUolzd/B/KHBcG1nAqSlirSVKQzx\nN5IrqaOljJhfQ/o/hGSbpxGCNiG6SlK6zs5OOKUHbn17Cg2t1jb2vtI5n1mNn7RJ8skkKYBwTv4E\nGhqrMLcn4TsD1kqSbRglajwejojBVsXl+Bc0DdnYvTSTNu3DcGkAYXpKIDPINSDoTrL7Rq89C/Ll\nEpOYp04Y5RqgUG+aViBAaX6YkiOVYj1Y/UArfrCyKlEjpjwLwCGUpF4p/sv3a3CdpNVOE+y598Ee\nUTgc7yKMsk5sOIqZfULS2yUthRBeN/zZrKQ7JO2V9KykG0MIJ8zMJH1U0tsk1SW9K4Tw4EZjhExQ\nZwG84eBk3qtBX4z2AajC8UFUI5SZExZoKSbJ91NtJhgV6MBTCl1YCShpEil0wueGT8bwu6OjPU2E\nU8efCqoBrgONZNE5S557YydbJ/GBjbJlwXcnUbhRIiKbmR2flHTdy372fklfDCFcJumLw79L0lsl\nXTb872ZJH9v8pTgcDofD4XilYUOfJYTwFTPb+7IfXy/pmuGfb5P0ZUnvG/78UyGEIOnrZjZtZosh\nhEMbXglx+hJMEQhqOoQODPGvx0dUqMARjkqAx2YFmNKC0RyrsudGDjlckIzZZ5eTiwpguWss5hFv\nmq6xL491QOi9BxwTXHZN04lA4p1GY2jpcoDzBumYkK++BeW7Cyc5F4clLQz/vEvSCyf93oHhz77P\nETGzmzWImigzPaPssXiuBbpXdGGD+UMqzNUtA2eChvwgWTWzFn/vYAUqB9zQUNk2fGy0zw5Z2KiG\nCc63Q55FaINNAXoChAsnial6SRI4NNGSc5rKJOs80biShNcKtMZLCoCblFonrXs3/6uYrBpCCGaj\n72ghhFsl3SpJ+Qt3h84UIJyC/GMGnmzbkMneg6p5hHCagmRVysIn5D1ScSPxDY0QNiWorJowxRxp\n51BRVygVLuJIiEU16KGHnOoHF5CcE0hbIlCSMolq4GaB8LHRSFofTLw+IPmOop8Su6QdeSnlYmaL\nkpaGPz8oac9Jv7d7+LMzIxgi9JBJRuV3M6twlqVgagaIitGFMQUXdbIp9WmvF7i20BNej4iCwXbu\nGagKS8LkNMydO8q8MOrEkdYAGSCqJZ2N/kjMnkTi+nAzb88lJ8aGU2p0zsHDMllreigduHnb2G94\nt6Sbhn++SdLnT/r5v7QBfkLS6qb4IQ6Hw+FwOF6R2Ez57mc0IKZuM7MDkj4o6RZJd5rZeyQ9J+nG\n4a9/QYPS3ac0KN9996auoi+libgVqPzFctk0xA/JDuSUE2BYgPY8IadLLLsMQbs2E6nypKM5qOty\nkhrrkrLrzD4NIpB9JjnEyeG02zbpWUIFCKH+DEmPZFeTJQnjLulkrQG9mc6qjkgI4R2n+adrT/G7\nQdIvb374AazPNiWSf6S5fmpPN7TORLwtaToncWeAqMIKqovmoCBZB1aukPmegwsjDfGTdCadc7gC\nAW4KnVK8LXFiJL4hNXewxYqkCCjZlDpxWTBvaPo+yZSYJJSW6oLD5ijv2lgoq4ZsUHMx3t1OAbJq\n6cXkJJ8l5khIkECWcKeh/DFA/IMLE8mbSpxrQMl7bGxmn1gTLeksCKIxe/LdMw02dh86Mvkl9uC7\npfibT2wlfmAjTSapEjGNPONIGkD+aPycGSXa703vHA6Hw+FwJIaxiIgomFKgNI3kwKimQuE4LGHN\nJxuyRIDtrYnMegqekGjzM6zFQQSeaFQB2qNSSqqBAtNKxaPJ6VG0YfSTRrJo1Q6JDODUSpXZk3lD\n0+80DNiaSy56Shr+jaKRNRaOiHWl/NF4R4Tk8Gjetj2VrCNByK50UcbN15qg/8P2BAmXktJwUUcl\nsDC9QMPkhCtAeQ44Xw7t0w2QnigkK4zVmobqpmCtwaqusDdUfjneNt2ijRaTfe6EV9WejB+3P4Lj\nPBaOiFJStwIEZ8Akb26DehSw6oWezNNkQ01a1CsLWnOvwLHhzO8C0qLEnGe6qBPtGUnK1uJtR1mc\nTgVEcJbUK9JNAUTx4Fph8LklqWrbnoKbOXSeyW5eoAc2eOgpHoLjg3lHOIyjUGPGwhEJxjYG8qCp\nfG+mjszxhojscQdbZp4m5D3aRBXe99wqHB+cUuhmXt/NPFASxSsuwdQIkcaX1C0ic5QSLL/I7nu3\nBNOJuQQ7PsM+OdSRQcqq8NpTXeiETSTniGwVnKzqcDgcDocjMYxFRMT6LOfeA2E7Gq4sHWHebnVX\ncqcUGs2hbiwhfFLiHo0q0JM1AS0DpVocRMiuW2Q3PruGzDFHhczZxjzsj0Q1i9q0ZD1+3hSP0fLd\nBKM5kMdHIyoU3XK8bQakYbe06d1ZQWBcCbI40Je7MQ85HjjvG29LuQb0BSXXTjmPlcNsYWxNJ5ci\noBVDxSO0ciR+4nQAF0ySuhVkrtyJ5HgSrRk0NK/egKq25LCYAtUXktSch1w+eu8A8ssw/Q+J8Qac\nCTLfzzuOCAURu6ETFAs0QZCJ0gGessQVB4nUOHYgYdUNjagkmbdtTTN7InmdZERC4vc9txY/56m8\nPCXadqATRw4ulN/SLbMXnlSpZdfZiQ1HT+GBkUS+KdF20+NszTAOh8PhcDgc34+xiIhYYFLppPdG\ntpqsIBk92RORJHq6pFwD8t1z67BhH6ztN+jC50/EX38Hsui7tE8OqFwxeMKi6cQe1GBpAd0gerqk\nKbk8tCepWMrpysLeUKRKjpa703WW7hFdMH4fVCuNUtE5Fo5IMJbiIKVdtKSN5u/o4kQUB+nYWJwK\n3Hrc0RKrJTJzsqhTdVHevTfelipkdoDAksTnLHlulGRMN3MiTkVB5xx1nklRQg6m1ERTcljAMN4W\niVaebxwR60tZkMciXWDpZkwl3lMdeLKvgK7FUK2QKg6SaBKNRNFTDhU0I2JulRfYblpvMy+usQAc\nf+hEkXYOEtftIaDvOhZEg92y21PxtlyEL8H3HTqvSVYrDcaPtyX3bZQDy1g4IgpCJ0xyUjAYrswA\nyWdJasyxN7S+CDaFEnvDSgfZtReW46+dVq2QRVU6CzLrYNq0+uy+47YCYNr0CrClOlWFhWWgxBlo\nzsGxafkujMjkQOk0dUSo45+pE0VcWGE3k6zzjN53cN9HSYc5WdXhcDgcDkdiQL6Wmf1bSb+oQTzj\nEUnvlrQo6XZJc5IekPTOEDZxjkkoIkLzrpRjYn2asI83xaJgMCqACKP0ttGcNex9QdJidGxSNi1J\n+ePx9q1taGgcUcGaDiCq0ITfXTQaBE/2RJiLcnNI519J6oOUIE2N5EHkV+JN81D7FBABHMU2+hLN\nbJekfyPptSGEhpndKennJb1N0kdCCLeb2Z9Ieo+kj53xs8TCjiRkiHuW0BpvSHad3B9v255INsRP\ncuaU30IVOqmADBGHogtTa4Y2eowfP1OFjgBsRUodUDJncyvs2mmlGD14hB6ooACcqMEHMHNCsE5D\nbk+uBt83OGeJ6CYZe5QlknJEMpKKZtbRIJt0SNKbJf3C8N9vk/QhbeCIKAhNNOLxUW81v87ekMYM\ncwYIaZOcEiQu5kacOEoAS7piiJCz25SwCaN4pKUC3VBKUBXWYFSAHBwMbOQSf9+as8k5cZQc3p5m\nE4eIktFSf8oNIoKdElyriO0I8yX66YQQDkr6fUnPa+CArGqQilkJIbwUlDkgadep7M3sZjO738zu\n7zaIoL3D4XA4HI7zFSQ1MyPpekkXSVqR9OeSrtusfQjhVkm3SlJp+56QIqcFcFSg6YV0C3rqsK05\nCbfS3CeNKuSAmBy9dl6bn1zVDg3VhhRtqR5vS1JSktSYZ/bFo5QjEh/jb1dgbgSCVKlJUhdIzNNo\nTv5Ecg0DqeglLZs2WLbdAZUvpC3AKFUzJDXzjyQ9E0I4KklmdpekN0qaNrPMMCqyW9LBjT7Ieuxh\nk8UtC8tvae6TCmuR8SeeZ04UbdhH+rV0JpNt8kOUGiXmTKCOmJJSsIS1A9R86X3rZ2l6AzqQgFdF\nORq0jBT3RyL8GNCjZzA2c0Ras/G2tLEpFfGjta3knUP70xYJmj0v6SfMrCSpIelaSfdL+pKkGzSo\nnLlJ0uc3/KQU4zoQrkELnqxl7O2mmzmpuulCUbAWdAbya1SaNR604oee8EgDtPxqcvdNknpFkG+H\njRJLh5IV4SPOBBU06xVoNAfyY4gDTLmq8H0l41P9FuK4S8IFFaTpXX4lfvBRIubRjkgI4V4z+6yk\nByV1JX1Tg1TLX0q63cz+4/BnH9/4w5jnhTYF6G3SUHOmyTaVXp60ZE+2m2cayC5jBRz4cmdpVAKc\nbmkUjp6MScVRbpWNTTuZ0jA3Iatm6jCiAdcaKqxFVHEpWZWCbMbYEaDEeDhnSa8bQtQdJUCAAqUh\nhA9K+uDLfrxf0lXkcx0Oh8PhcLwyMDYS7+SEmAYeJ26CBe8gEQmSWLi1C0O9SL9FUhbU19Nrp+FW\nCkL0pXMW0iRQeqMNutdKUreMzFHzM0lKVeNvPn5uyWbk8MmcgEbCSAns1LNssaAFDfVtbJPpEAFD\nYnreNb0LTBeCbObZGpskNO+Lqz/AhozbS8P+D83p+PwKve+Um0PTWm3Ar6HCVs1tyaV2qPNKG79R\nngR5X1Mwxk9VmGmvGcSzgOkJKvyYBs8dr/EwlVrfkZz+SxaSjDeLsXBEQkrqlJPZlNoVRjZoztJ+\n9MycCM5QUTBaRkqAq5Xgok7Lf0kJbOlocvLyktQDhNMerNih4k6UJ5E/Abg9MPpJpQYo2ZVUNlJi\nei/PiE1kvVh+NdsmqRNVXKIObLxtDkQAt0Ti/WyDEE5JmB6zsSFS0BkgjkxjO9QwodoAK8yeoA27\n99IS2j6IRlHnl147kXgnLHyJaVlIPArYmYgfn5A9JU74pKXTpGKItpNobkfmqKigdJhGX5l9B2oW\nkee+fkH8Qx/lsObddx0Oh8PhcCSGsYiI9DMsb03CtZWD7IhUXGLHlPoiOyZVF+N9yfY01BWoQ+Ih\n4JjQa6ddWAnRVpJSIDJAUytUWKsB9GdoFK78IjxdQsIn0TtKumN0CnJEUOM4GBVIUwVqcOTGkSxc\nMg7nfFJdzs83sqoMVp+Am9XYxoJCqS5L3FJRMVLBkKklWzVDdBV6sHEb3YzXL6Qd/wDpES5sVFE3\n04y/9vYkGlrV3cmqXJINjXf+ZfaU11QHFU9Z2HWZ8iQIaLUSFpKrswsIqfjNlewvo1QmjoUjEtJS\nZzL+C2fXEzyl4JI8yMjOxa+MRN1T4gsjIZDRDYWW71JRsG4FPHdafwtRWI6f9CEFu00nrIiLyuVh\nrp927zVSxilGlqVEW1r2nQWVZrQNSGsK9smhgmqN+Pc1txa/UI5SROIcEYfD4XA4HIlhLCIiklBk\nAjHh4QmpMcuOxjRFQOwbC8zVLhxLroIBlw7DiAiVeCcdOWnOGlfdgJRaYzuNACJznI5E3UizLHxa\neZY9t23fYrXPaxfG33yqXdOaQeYoGpR0KwwaRawvgqrSTPwG0/nW5scdC0ck3ZRmHo+375QB8Q86\nIqVjLGFPywnrnfhHSNVJiaaCxHhBdGGjaaXcKgyTA24QLXumc44QNkuH2Ng0lUkbBuZPxN+81jQ8\ntKTYtTe2s/wI6VlCu+9WDsL0CBAQ7MGS8SzpcyO+Vk08C7h44JmPUm4+Fo5Iqh1UORBffVLdHe+p\nc7VCtjhk11nVTXUx3t2mUQWqTlo5GO/EpYADJnGOCCEtSpIBR4ZG0aoXMHsSvUR8Lkk0hElbKqDm\nnJRXBDdEqkNC+HBJN70rgENTgIeODOz4TLWm2pPxi5UBJ2qU+TIWjohSUq8Qf7MKy/GrQ7rJdqRU\nD/YR2MmaKJANEZ/qIXmuPRW/MpeOsedGCFyS1CmzXYVsKrTSqreHST2WK/H2jSem0dilF6EQHeyA\nS54b6acl8U7fWTjnDVTJUUEzWimGDg7Qd6aORArWnKfboNwepHFHuedOVnU4HA6Hw5EYxiIi0sub\nVi6Jz1+SHgjZOvPFaPdd6umT9AqWy4aly8Vj8R9Ae800SuzB5VdYRKZbiI+o0DLQfptFcy6dPRZt\nu/x6pqr1Ym8R2Rdhnx5SykkjWSmYRqbvK0kt0ehpug0jmMX4dZ42Ju3Afmb9DCx5B2tl4QToNXO+\n6YikOlL5cPzCTpTnsuvME6jtZDR+pNSoZHPWVIOFfPckCZeS1JxjN49cf2kJtiX/GiMtfiu/O9p2\ncoIx9+icpZsxaZJJW9nTOU+br2VBWitNxdjgwSMP9DBIY1GJ85LooasPiLpkvo+SmhkLR8R6AQmn\n9LPxN6u2CB0JqPBJW0wTbStSQno2gLqRUicKAsusk5bqkGtATjmSVL4/fketzQIavqTKYahEDPPt\n9R2kSzgaWhOH2aGpC6ICkpSpkcMi86J6YI2XpG6FNG9jY3dg5JlWZhaPAwcSRKK2TNDMzKbN7LNm\n9riZPWZmV5vZrJndY2ZPDv8PK8AdDofD4XD8oIJGRD4q6a9CCDeYWU5SSdJvS/piCOEWM3u/pPdL\net+ZPqSXN63tjQ8Xk9IwKt+bq7ITFs0bkxbRNNTbJc2UJFkAnjoMM9MweWsaVm+A65/aD3kOVfbg\nm0CymmrX0HQg7dtROAZKISHHg0bhMEcFRBUyNXiqf+4osm9eMh9t29nOIiJ0j8muslBaaxuQtyBV\nZlvR9M7MpiS9SdK7JCmE0JbUNrPrJV0z/LXbJH1ZGzgi6VbQ5HPxE7UB8vVkI5ekHOhhIJ0FXYME\na/tJQyQpYeIfJAlTMTfSO+PoFez8UDrM8lqVF4kjA7k10BGhm3kKki7R2DCNSw9NnTLYkBeYHHBr\nbgeyzy/Hb+ZZ6LySlJYk1XeydCYRc0sBiYX+Q1ujrHqRpKOS/oeZXSHpAUnvlbQQQnhJP/GwpIVT\nGZvZzZJulqR8YRrVSpcPx9vSU0q3yBZWzDHpAsVANr+Vq0FlVXDraMVPYQXKbR9kctkHfzI+JDN9\n9RE09kyBVa48/uieaNuJJ2gUDZlrPcfe1xyo0EOcKEm1BahdAw9dqPMwbLhH7rskdSbjtzrkgIlz\nc3AUEZiTdhKj8OCII5KR9AZJvxpCuNfMPqpBGuZ7FxJCMDv10hFCuFXSrZJU3rYnrF0ASKOI+Bdv\nK0lt2IeAMNElFhGh8va0dJlgaKmWAAAgAElEQVREozqwnTwhOEtS6Qizn3om/sH99A1PorF/Zuoh\nZP+ZytXRtl868KNobEr4pPL4JKJS3wGrH+D7lgOCZBKTSaDVSlQUjJBlG7Psxqdp2TWtEAQp9DZY\nZ8MIt42spgckHQgh3Dv8+2c1cEyOmNmiJA3/vwTGcDgcDofD8QOMaFcvhHDYzF4ws1eHEL4r6VpJ\njw7/u0nSLcP/f36jz+rlpNpOkMcCHiMlPVKeBY1KkPRKn8lJqAV6GEhMEyLNsgvKr8BQ7wRsYAae\n+4Emk0mfnmFppasnn4q2/au5K9HYpUNszrUnqG4PaROOhsY6IBlImiTRY6r/QhsGZg7FbxKU2E4f\nPC3X79HUzhaAVs38qqRPDytm9kt6twZRljvN7D2SnpN040Yfku5I5UNbw859OSjXgC4u1BkglSvd\nEhubEnV7QASFtoNPw1BvIAIuYoJqX33yUjT2m0mra0lNMGl7M7Bb9QvswTfnkLkyTaBSeSzZED1t\n9Jhfjf8A3G8FkoRJUQBt7tmDaxUtaEA9xYhO1QivOnJEQggPSdp3in+6dqQP6rONoQka89C8K46o\nQI4KydtSJ4yQTSUpDRRxaSiptghL8iB5juTMJx5kLOMHX3shsv+F2a9H2/7m1X+Nxv79zluR/SV3\nsBdufU88e685k2yFHt0Qu6AxaSoDiwLgqZ5w8XLrkOACQQnazWmgjrpFwRRveudwOBwOhyMxjIXE\ne8iwqAbRs8hArkGapdtRnxxJaoPUTPEIG7u4THUJQFoJiqnR50bDtSQll4Vl0//r0cuR/b6rnom2\nvbzwAhr78h96Htkvz7JoUOFEfHqiupsttzTXXznI3lfGr6G9YmDTvGZyEg/tKfbcO/C5k5Te5Avx\nC90o+/JYOCL9jNSajbe3TvyDogQwSlZtzCfXs6Q9zV6w6oUsoJZbBc8NbsZU2IqGuQlHhHJ7pr/G\nxKV+J/Wz0bZvvORpNPbRehnZCwrhkU6sNA1MV2vcH4m87jTED1SYJdaksrqbrnNQ+LGGzJWGQnhb\ngbFwRKzPNnRyus0vs4fUgnlfSlYlCp/pFpSMhnsCYaPTRZVGwprbkjul0JMtFoP7TvyD+7v6q9HY\n6XVGTJqjnVCBD1c4hobGHBG61pA5SwXJ8idgwz8gT58/Afkp8LvTzsWk6qa2GD9pRukaPBaOiCRU\n+VI8Gm888whT+elOs9NlfYEdrYl8LxUZoumNAnhu+Nohi9+Aoq0kdYAQHiXulUEpoyRla/EnxA4M\nUxtclGmYvQBOt11QcSPxOUvbyfegPQHtkp5txC8YE8/DigJ422j5brcEyKppIqe7+V91sqrD4XA4\nHI7EMB4RkcC4GqSMtF9ktyB3pIrs25NMnKoNQo4TB5mnTzuZZtbjj7edSRZnbs2w505TO0SIjvYM\naYFyPomlxfLLCWoiiJPDWxPx9641myBHQ8k2uaQyBZRfU9tBUnqwPxGcsxMHWOg5BTiUW1W+OxaO\nSEhJ7Yl4e9JQqTPJQn60M2IHhM0kppZYBwQuSbJZKCQS4p2J4jJLL9BFnTSDouOvv54tTGtQmKD4\nOOyWCEC6FktSL5+cIi51BAonIDcIEnXzQE+DkHwl7kAWVuOvnd43qgNS28H2KCJkdz40vTtrsL6U\nqcfbEzJQc4YtTKTsWIKS0ZJKQK2xMc+unTYgI5O8U4ERDbiwUcJn88fiqfC/ccXforFXYdnNn65e\nE21bPMTeN7qZU+IgmfMZUEIq0VM9n/MEhWUYfa0x+1QTPDioorzy6gqyp2iTtWqLKradI+JwOBwO\nhyMxjEVEJFvta/Hv40+IrW3xR+sGTE/kV9gph/Z/qC3EXz9t5mQwPUFyp7SUEUdECmze7Jxbjba9\nrvwYGhtSTHT/6y+Itn32UdYnh3JzJoBAkyT1c/FnNxp9LR5na035EPvuuaX4NTpk2XfvVlh6Yv3i\n+KgErdCj/BhaJUdAU2qbxVg4It1ySkv74kUpSHkTDbHn4vcTSayMU4IdbGGYm5RcU7RmWDCvDgXJ\nqLrpwYcWo22PX8Y8wEvyjOh79ez+aNv9ucvQ2E2oXdOYh94zAO/4DEnKU+y7z5EyUihI1lhg194p\nJxf8p71qDDJGSTpy2/0nom2frW7+lD0WjkhISR1AVt0qr+1UIIxkiROZuuD9JGJoEndkyEmhl6de\nEHxu8JREqkfuXn0DGnvn7NeQ/cW5o9G26xexGzf3cLItzYmoFyVIY3vIdTj+uniSMq7YgXy0LuBX\nl49Abs9CclVqklQ+HH/zepX4DWaU6x4PR8TYyZ5I4E4/zeJmSbaDl6QUkO+tLcKqFyiwNPvwSrRt\na54RLus7WFSAijvlwOn4c0++Ho199ZVPIvs08MLCHA3DsZMxVRfNgPetuY1tSB0YDaJtEQhRN8Cd\npj4HxeBASi8DxNAkKdWBjgjsXNyeiF/nm7PxD67/2Oa/t5NVHQ6Hw+FwJIaxiIhYX0oDQTMS9usU\nYfktDNFToZ7WJLh+mN0oHmdM234x/njamGdH2yQJYJI0cTD+3vW+CvKYkv733iuQ/Q2z90Xb/ta+\nv0Zj//Hkm5B9+p4ZZC/wvpdgiD9TTzadSPrFFGCvmPXd8H0HXMAu3SPgOpuCrTTW9sZHREg6sDfC\nI8OOiJmlJd0v6WAI4e1mdpGk2yXNSXpA0jtDCGd2FYxtyO0d8U+asOAlFvKTpM4ks8+AtBRRSpSk\nNlCZlKR2JT69QsmqtINt5QAkoAH9GKqF8ZcPX47sL/yJ+O5try0cRGO/ZtsSsn+qx5SMCWGUKuLS\nOUsrjjJ1sM7CVOb006xNehcoaDe2sRR2G/QDk5J97jNPxDuQz49QmXg2IiLvlfSYpJe21N+V9JEQ\nwu1m9ieS3iPpY2f6AOux6pPcWvyDLh9mp/rcOrOvb2ePIA8UA3MnWL4+ZKEzUAQvOG4rzq59/QJm\nv3ZhvH3lAHNEKk+wkqE/1k9H277qosNo7AMrU8heUGadNHosLTHntQOb5jUhz2IdVJ40trP3ZeJ5\nutaAtgRgjZU4j7DVTy56u3Jx/P7U/foWkVXNbLekn5H0YUm/bmYm6c2SfmH4K7dJ+pA2cERSHakC\nOoI25kD/hyk2wYmOhyQZLGubeCGeQUY7kcJL19oF8dMvwEqpyWdZqHj5NcyBBOr2WF6eEGUlqfhM\nvCPzZHMXGjsFN+PUBOxgC+ZdG5bf0pLx4hFkruoe0LMEpheWf4i2k4g3za2zPSK3Rjt9I3N0aGuA\nbMMoBGVKVv0jSb+l72VO5ySthBBeunUHJJ1y5TGzm83sfjO7v9NijeMcDofD4XCcn4g+1pnZ2yUt\nhRAeMLNrRrUPIdwq6VZJKu7YE1Yvivd4iULn5HMsPbH0BnY8TcMTXmM+/nTaT7Oxa4uwHBFwLns5\n2EV1mkU0suvIHJ9uCWiPoMIyuHZjJ1tSqi9xfg2JRlXjBWnx2JJUOMbe9+LR+HtHZQpILzJJmnom\nPuLehN2qWzPwu9O1AgxfeSHedpQiErIav1HSz5nZ2yQVNOCIfFTStJllhlGR3ZI2ZKeFtNSaib/Z\nSJ3UWL6cviC0aqYOGteRSiVJam6DIUfQNbn8Iho6UUdAktZBmJu2BaBzlmj+EEEwibclp0rKpPIk\nU4UXT8UPYUsHUrXTh0rGdK1avTh+0haOsxufJ467pBRsjEqEJye/G0/e3F/bAmXVEMIHJH1AkoYR\nkd8MIfxzM/tzSTdoUDlzk6TPb/RZg+67hI0ORIYggYsy0TuwMWMPbOatvbBfCywnJBsiJYAVj7Gw\nwPoe5kF2y/H3nqiySjzn3IIVsASlI2zOrl3ExidzvrjErr0GnFeJl4ESR4g6kElGslZZVwJZl0ai\nqGhmvG1zJr7KrHNw887fuRA0e58GxNWnNOCMfPwcjOFwOBwOh+MHAGdF0CyE8GVJXx7+eb+kq0b7\nANZ3JLcKwtwwX05PhyENWfzAtjvPvnw4zkSGSIi/zeQglG6zqU/FocgRoAdD7HnAqZJYmJ1ELyVp\nDVZLkVJ/iaVS1/eiodUrwUnHdPAUMvGTNgvrEWhEpVMGFT8pmFqBj41ygwgXj/S5GUUhYSyUVVMd\nqXwo/ml1ARGK9kDotWivGEgY3RV/39LL7MvTpnet+XjCQLrOgnnVC2ApJdzQ0iAtRfPl9R3s2vPH\n421TbbijUI4I7NdCNiVSsi3xe0fnLHFAKReusT25LuXlg8k5r5JUByW0ktChh/RmGmVvHQtHpJ+V\najvi71YPdFYkuXqJd5VMwcUJncxpe+l1uCv041cHgyI/VHaZnDIkqVsBjeNgVIC8LxLbzLMgeinx\ngwPlxxCVy34WcrJq7N5VDiZH0Kan+l4OirnNx9vWdibLpSsdSk7QrDlPiEGb/9XxcETyUm0voNMD\nwubEM5AmA+dIG0q8F46CcCkshaQgCqG0E2lrOtlTTvaC+Ju/MMVqhw8cZfnEmXtAO/gOi1M3Z2Ep\n5RwyVwoQDye/zcbOwJYMhePMC6tvh6cmABrJIuX+hDYgsWiMBMvlJeXX4vfWue/E2x5e3/x1e/dd\nh8PhcDgciWEsIiIKQqVhqKTuGDuhrV4KG79NsfGLR+LHp6d6Gm4l/BiqBwFbzeCU3gd/5AvRtv+g\n8Bwa+5MLP47s73jqp6JtKSeKguqYtGbi39dl1mtQ+eO0ySSLaLSnwLNLuAMtEXOj6xzVUKHjdwvx\n82Z1L+g18/AW9Zo5W0g3pelHwYYKOjs2oI5I8TDNH7LFhXTfpS9Ie4oq/sXfe57rZ9fenYQNzEC8\nFgri6uryU8j+zte+IdrWHmTkmvwJ2PAPNrlcuQT0R6IheqBsKkltmI4k3KKku5QTDZfyIXbfq7to\np3D23EjHaKLqOgp/cSwcEeuz8q7q7nhbqlJJNlNJ+KRAOCpUMTC7hsxFvnwall3Pfpc9+Bf/IdtV\nPvL4tdG2X9t5CRr7xrlvIPufu/SRaNu7nr4ajU1JyusZuOSBVwZwsyVxkjEl1pNDD6mKlHgEk5D6\naan+9H621qztYROHqJaTZz4KnCPicDgcDocjMYxFRKSXl1ZeBT4A1Pbnj0M9iCaLKhDBGIlV4NIT\nGg31kvbYhRM0lMVAS/LWTsTXgX6pQV4W6crK88j+R8vPRNs++GN70NjP3QvCn5Jy8a0zJEk9EOam\nJeO0R1CmyY72bfDdia3EK1dI4zmaVsrWoUQE/O7Fw/HfPVuLnzPnXWom3ZKmno63JwS0XgGWRkGV\nysITLF7aBl1kO5DwSXKPktQAtf2NbWzq0tRM5Xk4b47HM9DSbUbuuWPbPmT/7j1/H237Mzvi0zqS\n9F92g0kjqXsM5jcAaPMzCkrwJloelB9jgd07cuihGibVXcmK+JF+aPUi4W5u/nfHwhGRmDNBlCJp\n5Qi3Z0z21mT8dy9Q4h/cjEnDQdwECy4uuXV2usw048fvQPLa0t/uQvb/6cq3Rtu+ZmEJjZ3JseNh\nfgXyogB5r0WqTiTVdiYrBkf6SeRhJCoDowrZRrw95WjQ+065PUSJOb8a/9BHaZ8yFo5IMFaiRE4a\nXdCDQOKplSZMb6SByBFtL01LMckppQyrHwpLrB5w+XVAYhNi8jnG1J04wJ7bMSAr+8gVbMkJKywa\nVHmROTIrl8YfHHBfKngypmkpgjRwBCSpOQcrT4AjM/UsmzP00FNbhOGk8wBOVnU4HA6Hw5EYxiIi\nIomVsQJ3igocUS2OiRfYBWTq8aGz5izztGnztNIRQDJeYVGBXhF234XzhoQ8+7DXTKfMzh8kklWD\nEQ3B8t2VS6BMOVinaIidlpHSVCxJDfUK7LkVjrFrbyG9KLZO0rWC2td2x9+79Yvix+1+afO/OxaO\nSKonFVZA911AwsIt0eEdbMNNobYQ/5JQJ4o0AJOkHhi/toPJDZYPs9RM+RDs27EQP3FoqDZ/gu1o\n00/H76hrl8BJB9MTqQ4UsiNrDXQEsCosTgMDY+hEUb0nUiFISb4L9wGRLEnNeUawNvDlQyp+fxpl\nvoyFI9LLSyuXxt8sIoY2cYBtKBNPMsWX3iRbmDvl+EfYyzMniIoMBSARSjeU6i7myFDiYWMBlIHC\nRbnE+KJKteIvIAe773YmKK8Jmau8Fr+j4iq1CrOnB4cSUJGmMuWk/FaSpp6Jf26V51j9bnqV2ZfX\n2aGpn52OtiXE+FHWKeeIOBwOh8PhSAzRx2kz2yPpU5IWNMic3hpC+KiZzUq6Q9JeSc9KujGEcOKM\nn9Vntc7Wj/fU2xXoi13I+lOThkSSlG6DiqE8DNWCsSXJQNVOA7LoqchQFkof94/H2/aKbOz6PLt3\n/XR8qJhKRluPCmMl2zeEgPIkFu5jN7++M37i0feVRlQItycNIxKrl88he0ENFZLKbYPWUKM8M5Ka\n6Ur6jRDCg2Y2IekBM7tH0rskfTGEcIuZvV/S+yW974yf1JcypLyL9BGAGkNUWTXVpkymeNPqTkbc\no+mR0lEQLj3IUmrtScazyLahSiUYv3KAjd0DTSIlliIoLbFrp1yBHuSqEoI1VgcFmkGStHIZOzSR\nXjc0nUi0MCQmqNadY54/TeOmW8nJJJCa8VHe1WhHJIRwSNKh4Z/XzewxSbskXS/pmuGv3Sbpy9rA\nETHoiLQm471tujg05tiG1pxlJ4XicbCwQyeM6hqs746/d+VDbGxKVl3fzY5o5IRHI1kFULEjSYXj\ngKx6ISPe1bfDSi3g/EpS4Vj8gtHPsXe9U2RrzShKl6cCcSaoxDtVZm3OxN/72g7miORgQQSN4pHv\nTsTYRtkfzgpZ1cz2SrpS0r2SFoZOiiQd1iB1cyqbmyXdLEn54rRy1WQ2VOqIpGFUoAeUGiXmwJUB\n6VCSWlNsYUVRMIjmLCMJUweSVAwRoqsk9WA6MH8ifvwi2MglqZdnSxZ14kgkq74dOhJwtablu8SZ\noERZmgptT8bb0urCHpxzDdjhnRQVpEnJ+QiXjROeZlaR9DlJvxZC+P8aw4cQgk7jJoQQbg0h7Ash\n7MvkWcjQ4XA4HA7H+QnkY5tZVgMn5NMhhLuGPz5iZoshhENmtihpw2LBfsZUn48/LRSILkLC6Qmq\nR5HqxH/37BpTWOpBB5KEeknpryRlV9l9Lx9G5iiiQiWjaQlrbUf8EZFeO+05QkHI5UTETuLtJEg7\nCEkSTK8Q0HWWaLjQLuPdUrJzNrcChOgoSXiTIFUzJunjkh4LIfzhSf90t6SbJN0y/P/nN/ysPmwm\nBTgiGUg2pWiC7rmSVFyOX9xqO6mGCTJHbc3LR5gj0ZpKlttDRJJoOpFWDBGFz9YsdUSQOXbCSKVY\nB/a1oqCO0MyT8RPv8FVsrWlPIXOVD8bbUo4HdSBzK8ic0QfWNv6V08FGmC5kF3yjpHdKesTMHhr+\n7Lc1cEDuNLP3SHpO0o0bfVBIMeEUwjWgXVSb02xDQzk4SVVQmtWFjgRtaz777fhZ3ppnSWcql42Y\n6JIab1mPtn3N9iNo7MeXTknb2jRS98Un3KkIHj0aZ2AIlKiLUo5GpsEmLSmXl6TGtvjtglbNUEVd\nIohGpCUkHs1px+uRSZIKQCqArpObBama+apOPz2ujf1ch8PhcDgcrxyMhcR7SEttUCPfAbnLPtRU\naMzTCgZkjurrC8fp6RCesBbjQzKUiU4rftZAMyhJetOeZ6Jtf3b2m2jsle0smvSx8k9F21a/yKIx\ntF9LYYUdzUkEdG0vLWGl4ofIHJ3s6diNKdj0bjbePlNnzw03veuy8fuET0cC/ltdvksRUlIHKLiR\nB53qsIdM0xNtKHaTX4kfn74glHhInEAcpobUoEaVpeS++Piro22f2rkNjX3Tnq8h+9+85J5o2wd2\n7EVj3/XkFci+8xWw0EgyoOmQA/l2ifPZaFqMvO+0uSg9eJDy4XSDqvkic57OBCBzbpQ1diwckWw9\naPsD8U+LCL7QCZ6CMucnJpjKUB1oShShZDQl/hFp/m6JapgwR2buO3RTiA+FHX5mFxr79y5/C7K/\n/uJHom0Xc6to7F2zzP5YhjkiKUBWoMJUKRhVyIOGfRJrTNqBm/Hs4+zUdOJV8ddONH8kqQDUeCVO\nciaiZNULQMXNCPdtLByRftaQ2A9J61DSYa6a7GZOHiBd2CiRaXUvuHpIAMutQUcmwWor2q8l9zdA\n3UnSXbveGG1bvhIw5yTVm6yesAzfVzLnVy9FQ+OTdX6ZRfEaO+K/fAtGnjsTUFoVPPbi0WSjOZSs\nStZKskeM8q54912Hw+FwOByJYSwiIr2sVNsZ77YhrgOsraocZCGNFkzNkIgKjWjQUHMuvoIVhyu7\noFxcYuWAktQrguZpVSpTnpyWR63BIhqlAgvjrcRTcyRJ008A2++ysZtzkJMFV/vikfhzK5V471Ro\nWiv+3lEdEHrfaQSUEIWJNMZW6YicPZjUyxMSWHKpmX6WBZXm7zuB7I+/YSbali4O5SNQg2U2fvrR\nPjWkg6zE88akX0safvcGbBxXWI63be+voLFXSmzOzQJHQmKhatqzhKrKEg0USSjETx13Wt1IDqu0\nspEWNNA0cAsow1INlM1iLByR3Is17f338Uz+9GR8zrtzxSXRtpLUnmIRjeZORp5DkwymXVdLcPqA\n94sok54NFOjiAqIKdGEkjoQktcGUza5CmfImVMSdQ+YqHQLKqjAKR7kClBOGoniUC0d5UeDASR0B\nyu1pw0NTAE3ziivxjn9qBOdvLByR/nRJjZ++Kto+U413d2uL7JjSZR2isdx29cL4iUJTM6UXWTSI\njE97IGSrzL4Jpcr7i/G2VPKZnsw7E/Fzlrai783BHa3HnlvtUhC5Pcy+fHadEj5hpRfwAXs52Mp+\nO1usJvaDYggosTBKiuKU9jSFDiTeSVpplGiKk1UdDofD4XAkhvGIiGRMNVC+q4V4WyrqRSMa6xcw\nX7BfBifEFLv27gptmpdcCWyKqhXCiExrPn7itZiemQRPWDYbH+MPcM6oCc9OOXiy3hafI6jmWPi0\n3YERyCzs/tuOHz9VZVtNZj25M3NxiZbvwguAyyRRHq8i1fLN/+5YOCKpblD5CFiYJ+MdEcpopo4M\n7SZafD4+3EtyvhIjGEtSyAAhulkW7+xVGNeAXLsk5WbitfnbJxhJJNVgi3o4Ee9MZFfY2IXjMD0B\nGz329sd/9xJVMoYbGpVZr+2Nf+dCljb8S84RoelEyjGpL7Dv3pqOH789Fz9p+yPsD2PhiHSLpmOX\nx18Kyl3CzTRFTyl0cSqA/B91JODaQL57+jhbHWjeNaShE7cevyNCH0jdCpt0BuY8jUR1WNGNGjtp\nwj4hW0kpGA0yKCpWOBS/Rk8+A7lwu5G5qhcA0iUo/ZWkTJ09ty48MFIncCvgHBGHw+FwOByJYSwi\nIlbqKbsvXk+j3Yn/Gv0OC9F3GuwWpovwhEbGTrOwALVPAY5KoHnTNntuoQ87oWbioxLkvklSJQcr\nTwDW52HtMUQKVs1k8/HvazbLIlEp2KmRnovXl+OjeN0ii2B2Z9mcLc3F58Bz4F2VpGabffd+i61V\nfRDBzJXi77uNELodC0ck1NNqfzNemCvTiB+bdEiWpNJhSPiEL2gAX6CxHQ2tPlwYW0AtkXZNJikt\nScofTy4l15qHeaVdUJQBIJ9nG0q7xd6XyjdgvT3A6uXQAYQOqDXYoSu/EL+Zt6AXlF1izz338FS0\nLRV+TMFdlrruJdB0r1OOJyalqptfI8+ZI2Jm10n6qKS0pD8NIdxyut9N9aQ8EBglddJUbZDqiFCh\nnzYg33VLcGGDi0u/EL+hhjkWSQowkkW5QYgAB/2QZpVVrqRA9QXQVhqMnWJffm1fPElYkrIH4hfm\nzDKbc1TCL1ODzvtqPEGn2IQtGWCFHdHOSbMpoyzYnySuSkvapxAnZpR16pw4ImaWlvTfJL1F0gFJ\n95nZ3SGER0/1++lW0NT++I1lfU+8p0/VRUlEQpI60JGpXky0i9nYmRrbjLMr8Te/Cx2BDFwYqSgY\nIaBlYSljew4u6u3455Y9wk62NJIVJpgD25kCpMcWVJWFjgTt19Kdil8w0hPsxNWj5b/V+HlHemJJ\noymMngr0wNcDr1xjG1DuHuGRnSuy6lWSngoh7A8htCXdLun6czSWw+FwOByO8xTnKjWzS9ILJ/39\ngKQfP/kXzOxmSTdLUj4/pdx6vMecrcaf0Gq74Ckl4dROAHnjzAwTFpi9lOmkp0GYPQtD9GRsSXrh\nGGv80e8C7Rt4Qtp30fPI/ocnDkXbHmnH94WSpB7swrV/nanB7X8x3p6SRbMVtti8btsSst9ZXI22\nbcAukX/75KuQfQqEBWhj1PIhFg3qpyE/BrSEINGcUaTtLdDyg1N9qNkNkq4LIfzi8O/vlPTjIYRf\nOc3vH5X03Bk+cpukY2f9Ql8Z8HsXB79vcfD7Fg+/d3Hw+xaPc3nvLgwhzG/mF89VROSgpD0n/X33\n8GenxEYXa2b3hxD2naVre0XB710c/L7Fwe9bPPzexcHvWzzG5d6dK47IfZIuM7OLzCwn6ecl3X2O\nxnI4HA6Hw3Ge4pxEREIIXTP7FUl/rUH57idCCN85F2M5HA6Hw+E4f3HOdERCCF+Q9IWz9HG3nqXP\neSXC710c/L7Fwe9bPPzexcHvWzzG4t6dE7Kqw+FwOBwOx2bgTe8cDofD4XAkBndEHA6Hw+FwJIax\nd0TM7Doz+66ZPWVm70/6es4XmNmzZvaImT1kZvcnfT3jDDP7hJktmdm3T/rZrJndY2ZPDv8f35Xx\nBxSnuW8fMrODw3n3kJm9LclrHEeY2R4z+5KZPWpm3zGz9w5/7nNuA5zh3vm8OwPMrGBm3zCzh4f3\n7XeGP7/IzO4d7q93DKtct/76xpkjMuxZ84RO6lkj6R2n61nj+B7M7FlJ+0IILvSzAczsTZKqkj4V\nQnjd8Gf/WdJyCOGWoQM8E0J4X5LXOW44zX37kKRqCOH3k7y2cYaZLUpaDCE8aGYTkh6Q9E8lvUs+\n586IM9y7G+Xz7rQwM4caA/sAAALBSURBVJNUDiFUzSwr6auS3ivp1yXdFUK43cz+RNLDIYSPbfX1\njXtExHvWOM45QghfkbT8sh9fL+m24Z9v02Cxc5yE09w3xwYIIRwKITw4/PO6pMc0aIvhc24DnOHe\nOc6AMMBLPTmyw/+CpDdL+uzw54nNuXF3RE7Vs8Yn3eYQJP2NmT0w7OvjGA0LIYSXmqoclrSQ5MWc\nZ/gVM/vWMHXj6YUzwMz2SrpS0r3yOTcSXnbvJJ93Z4SZpc3sIUlLku6R9LSklRDCS11hEttfx90R\nccTjJ0MIb5D0Vkm/PAyjOyIQBvnL8c1hjhc+JukSSa+XdEjSHyR7OeMLM6tI+pykXwshrJ38bz7n\nzoxT3DufdxsghNALIbxeg5YrV0l6TcKX9P8w7o7ISD1rHN9DCOHg8P9Lkv5Cg4nn2DyODPPRL+Wl\nWevSVwhCCEeGC15f0n+Xz7tTYpin/5ykT4cQ7hr+2OfcJnCqe+fzbvMIIaxI+pKkqyVNm9lLwqaJ\n7a/j7oh4z5oImFl5SOSSmZUl/WNJ3z6zleNluFvSTcM/3yTp8wley3mDlzbSIf6ZfN59H4bEwY9L\neiyE8Icn/ZPPuQ1wunvn8+7MMLN5M5se/rmoQQHIYxo4JDcMfy2xOTfWVTOSNCzD+iN9r2fNhxO+\npLGHmV2sQRREGsj4/5nft9PDzD4j6RoNWmIfkfRBSf9T0p2SLpD0nKQbQwhOzDwJp7lv12gQHg+S\nnpX0SyfxHhySzOwnJf2dpEck9Yc//m0NuA4+586AM9y7d8jn3WlhZj+iARk1rUEA4s4Qwn8Y7hW3\nS5qV9E1J/yKE0Nry6xt3R8ThcDgcDscPLsY9NeNwOBwOh+MHGO6IOBwOh8PhSAzuiDgcDofD4UgM\n7og4HA6Hw+FIDO6IOBwOh8PhSAzuiDgcDofD4UgM7og4HA6Hw+FIDP8XfgruwtGqT18AAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0fc07f72e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(9,3))\n",
    "plt.imshow(x_train_vec[65129,:,:,0], aspect=\"auto\", origin=\"lower\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('x_train_vec_dec10.npy', x_train_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('y_train_vec_dec10.npy', y_train_vec)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"weights\"></a>\n",
    "## 5. Set class weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "ind, counts = np.unique(y_train_vec, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 4232,  4210,  4230,  4190,  4212,  4222,  4220,  4202,  4268,\n",
       "        4224, 73636,  7200])"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_weight = {i: count for i, count in zip(ind, counts)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_weight = {ind: len(x_train_vec)/val for ind, val in class_weight.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 29.075141776937617,\n",
       " 1: 29.227078384798101,\n",
       " 2: 29.088888888888889,\n",
       " 3: 29.366587112171839,\n",
       " 4: 29.213200379867047,\n",
       " 5: 29.14400757934628,\n",
       " 6: 29.15781990521327,\n",
       " 7: 29.282722513089006,\n",
       " 8: 28.829896907216494,\n",
       " 9: 29.130208333333332,\n",
       " 10: 1.6710033135966103,\n",
       " 11: 17.089722222222221}"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_weight"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"cnn\"></a>\n",
    "## 6. Train CNN with Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "import keras.backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_labels = 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(128, 32, 1)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_vec[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 128, 32, 1)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 128, 32, 64)       10304     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 64, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 64, 16, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 64, 16, 64)        163904    \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 32, 8, 64)         0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 32, 8, 64)         256       \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 32, 8, 64)         41024     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 16, 4, 64)         0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 16, 4, 64)         256       \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 16, 4, 64)         16448     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2 (None, 8, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 8, 2, 64)          256       \n",
      "_________________________________________________________________\n",
      "conv2d_5 (Conv2D)            (None, 8, 2, 64)          8256      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_5 (MaxPooling2 (None, 4, 1, 64)          0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 4, 1, 64)          256       \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 500)               128500    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 250)               125250    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 250)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 12)                3012      \n",
      "=================================================================\n",
      "Total params: 497,978\n",
      "Trainable params: 497,338\n",
      "Non-trainable params: 640\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = keras.Sequential([\n",
    "    keras.layers.InputLayer(input_shape=(128,32,1)),\n",
    "    keras.layers.Conv2D(64, kernel_size=(20,8), strides=1, padding='same', activation='relu'),\n",
    "    keras.layers.MaxPool2D(pool_size=(2,2), padding='same'),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Conv2D(64, kernel_size=(10,4), padding='same', activation='relu'),\n",
    "    keras.layers.MaxPool2D(pool_size=(2,2), padding='same'),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Conv2D(64, kernel_size=(5,2), padding='same', activation='relu'),\n",
    "    keras.layers.MaxPool2D(pool_size=(2,2), padding='same'),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Conv2D(64, kernel_size=(2,2), padding='same', activation='relu'),\n",
    "    keras.layers.MaxPool2D(pool_size=(2,2), padding='same'),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Conv2D(64, kernel_size=(2,1), padding='same', activation='relu'),\n",
    "    keras.layers.MaxPool2D(pool_size=(2,2), padding='same'),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Flatten(),\n",
    "    keras.layers.Dense(500, activation=\"relu\"),\n",
    "    keras.layers.Dropout(.5),\n",
    "    keras.layers.Dense(250, activation=\"relu\"),\n",
    "    keras.layers.Dropout(.4),\n",
    "    keras.layers.Dense(num_labels, activation=\"softmax\")\n",
    "])\n",
    "\n",
    "model.compile(loss='sparse_categorical_crossentropy', \n",
    "               optimizer='nadam',\n",
    "               metrics=['accuracy'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.python.keras.callbacks import EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks = [EarlyStopping(monitor='val_loss',\n",
    "                           patience=5,\n",
    "                           verbose=1,\n",
    "                           min_delta=0.01,\n",
    "                           mode='min')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 123046 samples, validate on 6798 samples\n",
      "Epoch 1/50\n",
      "123046/123046 [==============================] - 187s 2ms/step - loss: 16.1380 - acc: 0.4163 - val_loss: 0.8133 - val_acc: 0.7224\n",
      "Epoch 2/50\n",
      "123046/123046 [==============================] - 179s 1ms/step - loss: 8.9434 - acc: 0.6521 - val_loss: 0.5923 - val_acc: 0.8008\n",
      "Epoch 3/50\n",
      "123046/123046 [==============================] - 179s 1ms/step - loss: 7.2433 - acc: 0.7221 - val_loss: 0.4908 - val_acc: 0.8455\n",
      "Epoch 4/50\n",
      "123046/123046 [==============================] - 179s 1ms/step - loss: 6.3200 - acc: 0.7556 - val_loss: 0.3619 - val_acc: 0.8773\n",
      "Epoch 5/50\n",
      "123046/123046 [==============================] - 179s 1ms/step - loss: 5.7280 - acc: 0.7768 - val_loss: 0.3583 - val_acc: 0.8792\n",
      "Epoch 6/50\n",
      "123046/123046 [==============================] - 179s 1ms/step - loss: 5.3283 - acc: 0.7951 - val_loss: 0.7126 - val_acc: 0.7673\n",
      "Epoch 7/50\n",
      "123046/123046 [==============================] - 179s 1ms/step - loss: 5.0806 - acc: 0.8017 - val_loss: 0.4203 - val_acc: 0.8632\n",
      "Epoch 8/50\n",
      "123046/123046 [==============================] - 179s 1ms/step - loss: 4.7601 - acc: 0.8128 - val_loss: 0.3924 - val_acc: 0.8708\n",
      "Epoch 9/50\n",
      "123046/123046 [==============================] - 179s 1ms/step - loss: 4.6085 - acc: 0.8168 - val_loss: 0.3272 - val_acc: 0.8953\n",
      "Epoch 10/50\n",
      "123046/123046 [==============================] - 179s 1ms/step - loss: 4.3559 - acc: 0.8254 - val_loss: 0.2609 - val_acc: 0.9207\n",
      "Epoch 11/50\n",
      "123046/123046 [==============================] - 179s 1ms/step - loss: 4.2292 - acc: 0.8294 - val_loss: 0.2818 - val_acc: 0.9086\n",
      "Epoch 12/50\n",
      "123046/123046 [==============================] - 179s 1ms/step - loss: 4.0398 - acc: 0.8349 - val_loss: 0.2563 - val_acc: 0.9225\n",
      "Epoch 13/50\n",
      "123046/123046 [==============================] - 179s 1ms/step - loss: 3.9775 - acc: 0.8368 - val_loss: 0.3110 - val_acc: 0.9044\n",
      "Epoch 14/50\n",
      "123046/123046 [==============================] - 179s 1ms/step - loss: 3.8206 - acc: 0.8409 - val_loss: 0.3058 - val_acc: 0.9038\n",
      "Epoch 15/50\n",
      "123046/123046 [==============================] - 179s 1ms/step - loss: 3.6617 - acc: 0.8457 - val_loss: 0.3333 - val_acc: 0.8989\n",
      "Epoch 16/50\n",
      "123046/123046 [==============================] - 179s 1ms/step - loss: 3.6433 - acc: 0.8472 - val_loss: 0.3305 - val_acc: 0.8950\n",
      "Epoch 00015: early stopping\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(x_train_vec, y_train_vec,\n",
    "                              batch_size = 256,\n",
    "                              epochs=50,\n",
    "                              verbose=1, \n",
    "                              callbacks=callbacks,\n",
    "                              validation_data=(x_val_vec, y_val_vec),\n",
    "                              class_weight = class_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_val_pred = model.predict(x_val_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  2.78599282e-06,   6.80228762e-09,   6.94566182e-10,\n",
       "         2.62370015e-10,   9.99930024e-01,   4.64455585e-09,\n",
       "         3.23929328e-10,   2.21292602e-08,   6.14031959e-12,\n",
       "         1.55445823e-10,   6.71811795e-05,   9.41062441e-22], dtype=float32)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_val_pred[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_val_pred2 = np.array([np.argmax(pred) for pred in y_val_pred])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_val_pred_label = np.vectorize(id2name.get)(y_val_pred2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.Series(y_val_pred_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "unknown    3701\n",
       "right       372\n",
       "no          344\n",
       "left        321\n",
       "go          309\n",
       "down        304\n",
       "up          300\n",
       "yes         295\n",
       "on          289\n",
       "stop        274\n",
       "off         272\n",
       "silence      17\n",
       "dtype: int64"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"predict\"></a>\n",
    "## 7. Predict on test files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_files = glob.glob('test/audio/*wav')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['test/audio/clip_bd6d0fb25.wav',\n",
       " 'test/audio/clip_3e7a56353.wav',\n",
       " 'test/audio/clip_c5884a6cb.wav',\n",
       " 'test/audio/clip_ebf0d1f7b.wav',\n",
       " 'test/audio/clip_2f714f052.wav',\n",
       " 'test/audio/clip_05bfe5e6a.wav',\n",
       " 'test/audio/clip_317b4a205.wav',\n",
       " 'test/audio/clip_292983fee.wav',\n",
       " 'test/audio/clip_e086bfc8e.wav',\n",
       " 'test/audio/clip_d0d7fa566.wav']"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_files[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test = wav2spec_val(test_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('x_test_processed_dec10_1.npy', x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x_test = np.load('x_test_processed_dec7_normal_1.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(158538, 128, 32, 1)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_pred = model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  7.25300363e-23,   1.21090929e-18,   2.44212151e-19,\n",
       "         3.69178681e-17,   3.61696769e-19,   2.43067486e-16,\n",
       "         9.99993563e-01,   1.40492750e-12,   7.19334570e-24,\n",
       "         7.29242097e-17,   6.39184236e-06,   4.95808185e-36], dtype=float32)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_pred[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_pred2 = np.array([np.argmax(pred) for pred in y_test_pred])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(158538,)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_pred2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_pred_label = np.vectorize(id2name.get)(y_test_pred2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_df = pd.Series(y_test_pred_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "unknown    64764\n",
       "silence    11985\n",
       "right      10235\n",
       "left        9886\n",
       "go          9394\n",
       "no          9063\n",
       "on          7794\n",
       "yes         7651\n",
       "up          7479\n",
       "off         7176\n",
       "stop        6772\n",
       "down        6339\n",
       "dtype: int64"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels_df.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(158538,)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_pred_label.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['no', 'on', 'no', 'right', 'off', 'yes', 'right', 'silence', 'off',\n",
       "       'right'],\n",
       "      dtype='<U7')"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_pred_label[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_files2 = [file.replace('test/audio/', \"\") for file in test_files]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['clip_bd6d0fb25.wav',\n",
       " 'clip_3e7a56353.wav',\n",
       " 'clip_c5884a6cb.wav',\n",
       " 'clip_ebf0d1f7b.wav',\n",
       " 'clip_2f714f052.wav',\n",
       " 'clip_05bfe5e6a.wav',\n",
       " 'clip_317b4a205.wav',\n",
       " 'clip_292983fee.wav',\n",
       " 'clip_e086bfc8e.wav',\n",
       " 'clip_d0d7fa566.wav']"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_files2[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Write predicted labels to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('ccmm_submission_dec10_1.csv', 'w') as fout:\n",
    "    fout.write('fname,label\\n')\n",
    "    for i in range(len(y_test_pred_label)):\n",
    "        fout.write('{},{}\\n'.format(test_files2[i], y_test_pred_label[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('model_dec10_5CNNs_melpower_noise.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}