Computer_vision/1_Lenet/Simple_implementation_of_CNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reference\n",
    "- https://github.com/AashishGit/CNN/blob/master/CovNet1-without%20pooling%20layer.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "import os as os\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = data.drop(['label'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = data.label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = pd.get_dummies(label, columns=['label'], drop_first=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train,x_cv,y_train,y_cv = train_test_split(train,target,test_size = 0.2, random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Reshaping the dataframe to image\n",
    "x_arr = np.array(x_train)\n",
    "x_cv_arr = np.array(x_cv)\n",
    "X = x_arr.reshape(48000,28,28,1)\n",
    "X_cv = x_cv_arr.reshape(12000,28,28,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def zero_pad(data, pad):\n",
    "    data_pad = np.pad(data,((0,0),(pad,pad),(pad,pad),(0,0)), 'constant')\n",
    "    \n",
    "    return data_pad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax_cost(out,y):\n",
    "    eout = np.exp(out, dtype=np.float)  \n",
    "    probs = eout/np.sum(eout, axis = 1)[:,None]\n",
    "    \n",
    "    p = np.sum(np.multiply(y,probs), axis = 1)\n",
    "    prob_label = np.argmax(np.array(probs), axis = 1)    ## taking out the arguments of max values\n",
    "    cost = -np.log(p)    ## (Only data loss. No regularised loss)\n",
    "    \n",
    "    return p,cost,probs,label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv_net(input_data, Y, W1, b1, theta3,bias3):\n",
    "    \n",
    "    ####Forward propagation\n",
    "    \n",
    "    ## Input shape\n",
    "    m, n_Hi, n_Wi, n_Ci = input_data.shape\n",
    "    \n",
    "    \n",
    "    l1 = len(W1)    \n",
    "    \n",
    "    (f, f, _) = W1[0].shape   \n",
    "    pad = 1\n",
    "    \n",
    "    \n",
    "    n_H1 = n_Hi+(2*pad)-f + 1\n",
    "    n_W1 = n_Wi+(2*pad)-f + 1\n",
    "   \n",
    "\n",
    "    conv1 = np.zeros((m,n_H1,n_W1,l1))\n",
    "    \n",
    "    input_pad = zero_pad(input_data,pad)\n",
    "    \n",
    "    \n",
    "    for i in range(0,m):                                                  \n",
    "        for j in range(0,l1):                                             \n",
    "            for x in range(0,n_H1):                                       \n",
    "                for y in range(0,n_W1):                                   \n",
    "                    conv1[i,x,y,j] = np.sum(input_pad[i,x:x+f,y:y+f]*W1[j])+b1[j]\n",
    "        conv1[i,:,:,:][conv1[i,:,:,:] <= 0] = 0                           \n",
    "        \n",
    "\n",
    "    fc1 = conv1.reshape(m,int((n_H1)*(n_W1)*l1))\n",
    "    \n",
    "    out = np.dot(fc1,theta3) + bias3\n",
    "        \n",
    "    p, cost, probs, prob_label = softmax_cost(out, Y)   ## change it to y_train or batch\n",
    "    \n",
    "    acc = []\n",
    "    for i in range(0,len(Y)):\n",
    "        if prob_label[i]==np.argmax(np.array(Y)[i,:]):\n",
    "            acc.append(1)\n",
    "        else:\n",
    "            acc.append(0)\n",
    "    \n",
    "    d_out = probs - Y\n",
    "\n",
    "    dtheta3 = np.dot(d_out.T, fc1)\n",
    "    dbias3 = np.mean(d_out, axis = 0).reshape(1,10)    \n",
    "\n",
    "    dfc1 = np.dot(theta3,d_out.T)\n",
    "    \n",
    "    dconv1 = dfc1.T.reshape((m, int(n_H1), int(n_W1), l1))\n",
    "\n",
    "    \n",
    "    dW1_stack = {}\n",
    "    db1_stack = {}\n",
    "    \n",
    "    dW1_stack = np.zeros((m,l1,f,f,1))\n",
    "    db1_stack = np.zeros((m,l1,1))\n",
    "\n",
    "    bW1 = {}\n",
    "    dW1 = np.zeros((l1,f,f,1))\n",
    "    db1 = np.zeros((l1,1))\n",
    "    \n",
    "    \n",
    "    for i in range(0,m):            \n",
    "        for c in range(0,l1):\n",
    "            for x in range(0,n_H1):\n",
    "                for y in range(0,n_W1):\n",
    "                    dW1_stack[i,:,:,c] += dconv1[i,x,y,c]*input_pad[i,x:x+f,y:y+f,:]\n",
    "\n",
    "            db1_stack[i,c] = np.sum(dconv1[i,:,:,c])\n",
    "            \n",
    "        dW1 = np.mean(dW1_stack, axis = 0)\n",
    "        db1 = np.mean(db1_stack, axis = 0)\n",
    "\n",
    "        \n",
    "    return dW1, db1, dtheta3, dbias3, cost, probs, prob_label, acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimizer(batch,learning_rate,W1,b1,theta3,bias3):\n",
    "    \n",
    "    X = batch[:,0:-10]\n",
    "    X = X.reshape(len(batch), w, w, l)\n",
    "    Y = batch[:,784:794]\n",
    "\n",
    "    batch_size = len(batch)\n",
    "    \n",
    "    bW1 = {}\n",
    "    dW1 = np.zeros((l1,f,f,1))\n",
    "    db1 = np.zeros((l1,1))\n",
    "    \n",
    "    dtheta3 = np.zeros(theta3.shape)\n",
    "    dbias3 = np.zeros(bias3.shape)\n",
    "    \n",
    "    grads = conv_net(X,Y,W1,b1,theta3,bias3)\n",
    "    [dW1, db1, dtheta3, dbias3, cost_, probs_, prob_label, acc_] = grads\n",
    "    \n",
    "    W1 = W1-learning_rate*(dW1)\n",
    "    b1 = b1-learning_rate*(db1)\n",
    "    theta3 = theta3-learning_rate*(dtheta3.T)\n",
    "    bias3 = bias3-learning_rate*(dbias3)\n",
    "    \n",
    "    batch_cost = np.mean(cost_)\n",
    "    batch_accuracy = sum(acc_)/len(acc_)\n",
    "    \n",
    "    return W1, b1, theta3, bias3, batch_cost, acc_, batch_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "W1 = 0.1*np.random.rand(3,3,3,1)\n",
    "b1 = 0.1*np.random.rand(3,1)\n",
    "\n",
    "theta3 = 0.1*np.random.rand(2352,10)\n",
    "bias3 = 0.1*np.random.rand(1,10)\n",
    "\n",
    "x_arr -= int(np.mean(x_arr))\n",
    "x_arr = x_arr.astype(float)\n",
    "x_arr /= int(np.std(x_arr))\n",
    "\n",
    "train_data = np.hstack((x_arr,np.array(y_train))) \n",
    "t = train_data[0:320]      \n",
    "\n",
    "x_cv_arr -= int(np.mean(x_cv_arr))\n",
    "x_cv_arr = x_cv_arr.astype(float)\n",
    "x_cv_arr /= int(np.std(x_cv_arr))\n",
    "\n",
    "cv_data = np.hstack((x_cv_arr,np.array(y_cv)))\n",
    "\n",
    "test_data = x_cv_arr[0:100]      \n",
    "\n",
    "Y_cv = np.array(y_cv)[0:100]\n",
    "\n",
    "np.random.shuffle(train_data)\n",
    "\n",
    "learning_rate = 0.001\n",
    "batch_size = 32\n",
    "num_epochs = 10\n",
    "num_images = len(t) \n",
    "w = 28\n",
    "l = 1\n",
    "l1 = len(W1)  \n",
    "f = len(W1[0]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main_init(train_data,W1,b1,theta3,bias3):\n",
    "    cost = []\n",
    "    accuracy = []\n",
    "    for epoch in range(0, num_epochs):\n",
    "        batches = [train_data[k:k + batch_size] for k in range(0, len(train_data), batch_size)]\n",
    "        x=0\n",
    "        i = 1\n",
    "        for batch in batches:\n",
    "            \n",
    "            output = optimizer(batch,learning_rate,W1,b1,theta3,bias3)\n",
    "            [W1, b1, theta3, bias3, batch_cost,acc_,batch_acc] = output\n",
    "            \n",
    "            \n",
    "            cost.append(batch_cost)\n",
    "            accuracy.append(batch_acc)\n",
    "\n",
    "            print ('ep:%d, batch_num = %f, batch_cost = %f, batch_acc = %f' %(epoch,i,batch_cost,batch_acc)) \n",
    "            i+=1\n",
    "    return W1,b1,theta3,bias3,cost,accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep:0, batch_num = 1.000000, batch_cost = 2.355397, batch_acc = 0.187500\n",
      "ep:0, batch_num = 2.000000, batch_cost = 2.289821, batch_acc = 0.125000\n",
      "ep:0, batch_num = 3.000000, batch_cost = 1.992886, batch_acc = 0.312500\n",
      "ep:0, batch_num = 4.000000, batch_cost = 1.835800, batch_acc = 0.343750\n",
      "ep:0, batch_num = 5.000000, batch_cost = 1.992097, batch_acc = 0.375000\n",
      "ep:0, batch_num = 6.000000, batch_cost = 1.898046, batch_acc = 0.406250\n",
      "ep:0, batch_num = 7.000000, batch_cost = 1.614997, batch_acc = 0.531250\n",
      "ep:0, batch_num = 8.000000, batch_cost = 1.578523, batch_acc = 0.656250\n",
      "ep:0, batch_num = 9.000000, batch_cost = 1.647839, batch_acc = 0.593750\n",
      "ep:0, batch_num = 10.000000, batch_cost = 1.502046, batch_acc = 0.625000\n",
      "ep:1, batch_num = 1.000000, batch_cost = 1.537722, batch_acc = 0.593750\n",
      "ep:1, batch_num = 2.000000, batch_cost = 1.299085, batch_acc = 0.718750\n",
      "ep:1, batch_num = 3.000000, batch_cost = 1.160947, batch_acc = 0.875000\n",
      "ep:1, batch_num = 4.000000, batch_cost = 1.078016, batch_acc = 0.812500\n",
      "ep:1, batch_num = 5.000000, batch_cost = 1.270836, batch_acc = 0.718750\n",
      "ep:1, batch_num = 6.000000, batch_cost = 1.281610, batch_acc = 0.656250\n",
      "ep:1, batch_num = 7.000000, batch_cost = 1.007812, batch_acc = 0.781250\n",
      "ep:1, batch_num = 8.000000, batch_cost = 1.019251, batch_acc = 0.812500\n",
      "ep:1, batch_num = 9.000000, batch_cost = 1.097646, batch_acc = 0.781250\n",
      "ep:1, batch_num = 10.000000, batch_cost = 0.958147, batch_acc = 0.812500\n",
      "ep:2, batch_num = 1.000000, batch_cost = 1.113160, batch_acc = 0.687500\n",
      "ep:2, batch_num = 2.000000, batch_cost = 0.898343, batch_acc = 0.812500\n",
      "ep:2, batch_num = 3.000000, batch_cost = 0.737994, batch_acc = 0.937500\n",
      "ep:2, batch_num = 4.000000, batch_cost = 0.714906, batch_acc = 0.906250\n",
      "ep:2, batch_num = 5.000000, batch_cost = 0.941324, batch_acc = 0.718750\n",
      "ep:2, batch_num = 6.000000, batch_cost = 0.944641, batch_acc = 0.750000\n",
      "ep:2, batch_num = 7.000000, batch_cost = 0.711106, batch_acc = 0.812500\n",
      "ep:2, batch_num = 8.000000, batch_cost = 0.715499, batch_acc = 0.812500\n",
      "ep:2, batch_num = 9.000000, batch_cost = 0.816087, batch_acc = 0.812500\n",
      "ep:2, batch_num = 10.000000, batch_cost = 0.681727, batch_acc = 0.843750\n",
      "ep:3, batch_num = 1.000000, batch_cost = 0.868683, batch_acc = 0.750000\n",
      "ep:3, batch_num = 2.000000, batch_cost = 0.696449, batch_acc = 0.843750\n",
      "ep:3, batch_num = 3.000000, batch_cost = 0.537727, batch_acc = 0.875000\n",
      "ep:3, batch_num = 4.000000, batch_cost = 0.527936, batch_acc = 0.906250\n",
      "ep:3, batch_num = 5.000000, batch_cost = 0.770233, batch_acc = 0.750000\n",
      "ep:3, batch_num = 6.000000, batch_cost = 0.747199, batch_acc = 0.812500\n",
      "ep:3, batch_num = 7.000000, batch_cost = 0.541179, batch_acc = 0.906250\n",
      "ep:3, batch_num = 8.000000, batch_cost = 0.540304, batch_acc = 0.875000\n",
      "ep:3, batch_num = 9.000000, batch_cost = 0.656920, batch_acc = 0.843750\n",
      "ep:3, batch_num = 10.000000, batch_cost = 0.530768, batch_acc = 0.937500\n",
      "ep:4, batch_num = 1.000000, batch_cost = 0.714981, batch_acc = 0.843750\n",
      "ep:4, batch_num = 2.000000, batch_cost = 0.578889, batch_acc = 0.843750\n",
      "ep:4, batch_num = 3.000000, batch_cost = 0.426635, batch_acc = 0.906250\n",
      "ep:4, batch_num = 4.000000, batch_cost = 0.415489, batch_acc = 0.937500\n",
      "ep:4, batch_num = 5.000000, batch_cost = 0.665791, batch_acc = 0.812500\n",
      "ep:4, batch_num = 6.000000, batch_cost = 0.616087, batch_acc = 0.843750\n",
      "ep:4, batch_num = 7.000000, batch_cost = 0.432774, batch_acc = 0.937500\n",
      "ep:4, batch_num = 8.000000, batch_cost = 0.429917, batch_acc = 0.875000\n",
      "ep:4, batch_num = 9.000000, batch_cost = 0.552824, batch_acc = 0.843750\n",
      "ep:4, batch_num = 10.000000, batch_cost = 0.439480, batch_acc = 0.937500\n",
      "ep:5, batch_num = 1.000000, batch_cost = 0.609601, batch_acc = 0.875000\n",
      "ep:5, batch_num = 2.000000, batch_cost = 0.499782, batch_acc = 0.843750\n",
      "ep:5, batch_num = 3.000000, batch_cost = 0.354181, batch_acc = 0.906250\n",
      "ep:5, batch_num = 4.000000, batch_cost = 0.339322, batch_acc = 0.968750\n",
      "ep:5, batch_num = 5.000000, batch_cost = 0.594196, batch_acc = 0.843750\n",
      "ep:5, batch_num = 6.000000, batch_cost = 0.520852, batch_acc = 0.906250\n",
      "ep:5, batch_num = 7.000000, batch_cost = 0.359080, batch_acc = 0.937500\n",
      "ep:5, batch_num = 8.000000, batch_cost = 0.355314, batch_acc = 0.937500\n",
      "ep:5, batch_num = 9.000000, batch_cost = 0.477664, batch_acc = 0.875000\n",
      "ep:5, batch_num = 10.000000, batch_cost = 0.378870, batch_acc = 0.937500\n",
      "ep:6, batch_num = 1.000000, batch_cost = 0.532208, batch_acc = 0.906250\n",
      "ep:6, batch_num = 2.000000, batch_cost = 0.441131, batch_acc = 0.875000\n",
      "ep:6, batch_num = 3.000000, batch_cost = 0.301653, batch_acc = 0.937500\n",
      "ep:6, batch_num = 4.000000, batch_cost = 0.284218, batch_acc = 0.968750\n",
      "ep:6, batch_num = 5.000000, batch_cost = 0.541000, batch_acc = 0.843750\n",
      "ep:6, batch_num = 6.000000, batch_cost = 0.447890, batch_acc = 0.937500\n",
      "ep:6, batch_num = 7.000000, batch_cost = 0.306390, batch_acc = 0.968750\n",
      "ep:6, batch_num = 8.000000, batch_cost = 0.301652, batch_acc = 0.937500\n",
      "ep:6, batch_num = 9.000000, batch_cost = 0.420135, batch_acc = 0.875000\n",
      "ep:6, batch_num = 10.000000, batch_cost = 0.335491, batch_acc = 0.937500\n",
      "ep:7, batch_num = 1.000000, batch_cost = 0.471835, batch_acc = 0.906250\n",
      "ep:7, batch_num = 2.000000, batch_cost = 0.395121, batch_acc = 0.906250\n",
      "ep:7, batch_num = 3.000000, batch_cost = 0.261136, batch_acc = 1.000000\n",
      "ep:7, batch_num = 4.000000, batch_cost = 0.242709, batch_acc = 1.000000\n",
      "ep:7, batch_num = 5.000000, batch_cost = 0.499080, batch_acc = 0.875000\n",
      "ep:7, batch_num = 6.000000, batch_cost = 0.390150, batch_acc = 0.968750\n",
      "ep:7, batch_num = 7.000000, batch_cost = 0.266959, batch_acc = 1.000000\n",
      "ep:7, batch_num = 8.000000, batch_cost = 0.260931, batch_acc = 1.000000\n",
      "ep:7, batch_num = 9.000000, batch_cost = 0.374430, batch_acc = 0.875000\n",
      "ep:7, batch_num = 10.000000, batch_cost = 0.302506, batch_acc = 0.937500\n",
      "ep:8, batch_num = 1.000000, batch_cost = 0.422296, batch_acc = 0.906250\n",
      "ep:8, batch_num = 2.000000, batch_cost = 0.357760, batch_acc = 0.906250\n",
      "ep:8, batch_num = 3.000000, batch_cost = 0.228689, batch_acc = 1.000000\n",
      "ep:8, batch_num = 4.000000, batch_cost = 0.210559, batch_acc = 1.000000\n",
      "ep:8, batch_num = 5.000000, batch_cost = 0.464528, batch_acc = 0.875000\n",
      "ep:8, batch_num = 6.000000, batch_cost = 0.343516, batch_acc = 0.968750\n",
      "ep:8, batch_num = 7.000000, batch_cost = 0.236281, batch_acc = 1.000000\n",
      "ep:8, batch_num = 8.000000, batch_cost = 0.228747, batch_acc = 1.000000\n",
      "ep:8, batch_num = 9.000000, batch_cost = 0.337118, batch_acc = 0.875000\n",
      "ep:8, batch_num = 10.000000, batch_cost = 0.276132, batch_acc = 0.937500\n",
      "ep:9, batch_num = 1.000000, batch_cost = 0.380102, batch_acc = 0.906250\n",
      "ep:9, batch_num = 2.000000, batch_cost = 0.326681, batch_acc = 0.906250\n",
      "ep:9, batch_num = 3.000000, batch_cost = 0.202070, batch_acc = 1.000000\n",
      "ep:9, batch_num = 4.000000, batch_cost = 0.185109, batch_acc = 1.000000\n",
      "ep:9, batch_num = 5.000000, batch_cost = 0.435026, batch_acc = 0.875000\n",
      "ep:9, batch_num = 6.000000, batch_cost = 0.305291, batch_acc = 1.000000\n",
      "ep:9, batch_num = 7.000000, batch_cost = 0.211653, batch_acc = 1.000000\n",
      "ep:9, batch_num = 8.000000, batch_cost = 0.202573, batch_acc = 1.000000\n",
      "ep:9, batch_num = 9.000000, batch_cost = 0.305970, batch_acc = 0.906250\n",
      "ep:9, batch_num = 10.000000, batch_cost = 0.254137, batch_acc = 0.937500\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA9W0lEQVR4nO3dd3xc1Zn4/88zo1Hv1baqG264F2zAgAlJgJAACaSHJcmGZJckm93ktxuSzWazu9lN2bBs2kIKpJDyTQghhE6MwTT33pssWcWS1XsZzfn9ce8djaQZaQQaSZ553q+XXmjulHuuse8z5zznPEeMMSillIpdrqlugFJKqamlgUAppWKcBgKllIpxGgiUUirGaSBQSqkYFzfVDRiv3NxcU1ZWNtXNUEqpi8ru3bsbjDF5wZ676AJBWVkZu3btmupmKKXURUVEKkI9p0NDSikV4zQQKKVUjNNAoJRSMU4DgVJKxTgNBEopFeM0ECilVIyLqUDgHfBNdROUUmraiZlA8NsdlVz1rS0cP98+1U1RSqlpJWYCwZHaNmpae/if509MdVOUUmpaiZlAcPemeSTEuXjm8HkOVbdOdXOUUmraiJlAUJCeyB0bSgG4V3sFSinlFzOBAOBTV88lOd7NC8fq2V3RPNXNUUqpaSGmAkFOagIfvaIMgHufPz61jVFKqWkipgIBwF0b55KWGMerpxrZdqZxqpujlFJTLuYCQUayhw9eVgLAi8cvTHFrlFJq6sVcIACYn58GQG1r9xS3RCmlpl5MBoJZGYkA1Lb0THFLlFJq6sVkIJiZmQRAjfYIlFIqRgOB3SOoa+vB5zNT3BqllJpaMRkIEj1uslPi6R8wNHT0TnVzlFJqSsVkIIDBXkFNq+YJlFKxLYYDgZUnqG3RPIFSKrbFbCAozNQegVJKQQwHAv/MIe0RKKViXOwGAmctgU4hVUrFuJgNBLP8PQIdGlJKxbaYDQTaI1BKKUvMBoKC9EREoL69l37d1F4pFcNiNhB43C7y0xIwxlphrJRSsSpmAwEErCXQKaRKqRgW04FglrOWQKeQKqViWGwHAu0RKKVUbAcCXVSmlFIxHgicDWp0LYFSKpbFdCBwegS6lkApFctiOhD4t6zUHIFSKobFdCDITU3A4xaaOvvo6R+Y6uYopdSUiFggEJFiEdkiIkdF5LCI/F2Q14iIfFdETonIARFZFan2BONyCQXp2itQSsW2SPYIvMDnjTGLgPXA3SKyeNhrbgDm2z93Af8XwfYENUs3qFFKxbiIBQJjTK0xZo/9eztwFCgc9rKbgV8YyzYgU0RmRqpNwczUDWqUUjFuUnIEIlIGrAS2D3uqEDgX8LiKkcEiopxy1MN7BH1eH994+hi7K5onszlKKTXpIh4IRCQV+APwOWNM2/Cng7zFBPmMu0Rkl4jsunDhwoS2rzgrGYDTFzqGHH/m8Hnuf+k0//jIfowZ0SSllIoaEQ0EIuLBCgK/MsY8GuQlVUBxwOMioGb4i4wxPzLGrDHGrMnLy5vQNi4rygBg37mWIcf3Vlo9gdMXOjlcMzx+KaVU9IjkrCEBfgocNcbcG+JljwN32LOH1gOtxpjaSLUpmIUz0kj0uDjb2EVzZ5//eGBg+OPe6slsklJKTapI9giuAD4CXCsi++yfG0XkUyLyKfs1TwFngFPAj4G/jWB7gopzu1haaPcKqloA6PUOcLh6sBfw+P4aBnw6PKSUik5xkfpgY8wrBM8BBL7GAHdHqg3hWlGcyc6zzeyrbGHTgnyO1rbTN+Bjbl4KPgPlDZ28drqBjfMndlhKKaWmg5heWexYUZwFDA4H7bPzAyuKs7hlhTWJSYeHlFLRSgMBsKIkE4D9VS0YY/wBYWVJJresnAXAs4fO09XnnaIWKqVU5GggwCo+l5eWQEtXP2cbu9hrB4IVxZmU5qSwqiSTzr4Bnj9SN7UNVUqpCNBAAIgIK4ozAdhyrJ6Kxi4SPS4WzkgD4NaV1vDQn/ePmNmqlFIXPQ0ENicQ/OL1swAsK8wkzm398VwxLxeA43XtU9E0pZSKKA0EtpV2IDjb2AUM5g1gsAzF+dYenUaqlIo6GghsS4sykIDJrk5gAEj0uMlNTaB/wHChvXfyG6eUUhGkgcCWluhhfn6q/3FgjwCg0K5SWq3lqpVSUUYDQQAnT1CQnsBMe58CR2GW9VgDgVIq2mggCLC61FpYtqY0e8RzzgY21c0aCJRS0SViJSYuRu9eVUR7j5e3L5kx4jmnR1CjPQKlVJTRQBDA43bx1xvnBH3OmTmkQ0NKqWijQ0NhKszUHoFSKjppIAiTEwg0R6CUijYaCMKUmewhOd5Ne6+Xtp7+qW6OUkpNGA0EYRKRwTyB9gqUUlFEA8E4aJ5AKRWNNBCMg84cUkpFIw0E41CUpUNDSqnoo4FgHGZpvSGlVBTSQDAOhZnJgAYCpVR00UAwDk6PQJPFSqloooFgHGakJ+ISqG/vpc/rm+rmKKXUhNBAMA5xbhcz0hMxxtqtTCmlooEGgnFyqpBWtXRNcUuUUmpiaCAYp9FWF9e0dGsiWSl10dFAME6Dq4uHDg21dvVz43df5l3fe4Ve78BUNE0ppd4QDQTjNLi6eOjQ0IOvltPS1U9jZx+HqlunomlKKfWGaCAYp8GdygZ7BO09/Tz0arn/8a6zzZPeLqWUeqM0EIxTsR0I9le1cPpCBwC/eL2Cth4viR7rj3OnBgKl1EVkzEAgIleIyPMickJEzohIuYicmYzGTUdz81K5ZkEe7T1e7vjpDk5f6OCnr1i9ga/ctBiAPZXNGGOmsplKKRW2cHoEPwXuBa4E1gJr7P/GJBHhhx9axcqSTKpburnpu6/Q1NnHiuJMPriuhLy0BJo6+zjT0DnVTVVKqbCEEwhajTFPG2PqjTGNzk/EWzaNJcfH8dCda1lQkEZ3vzVD6DPXzkNEWFOaBcBuHR5SSl0kwgkEW0Tk2yKyQURWOT8Rb9k0l5kczy8+vo6FM9LYtCCPaxfmA7DaDgQ7zzZNZfOUUipscWG85jL7v2sCjhng2olvzsWlID2RZz531ZBja8uyAdhdoT0CpdTFYcxAYIzZNBkNiRaLZ6WT5HFzpqGTxo5eclITaO3up6mzj9m5KVPdPKWUGiGcWUMZInKviOyyf74jIhmT0biLkcftYnmx9cezu6KZc01dXH/fVt72Py9R36aF6pRS0084OYIHgXbgvfZPG/DQWG8SkQdFpF5EDoV4/hoRaRWRffbPv4yn4dPZmlJreOjpQ+f50E+2U9vaQ/+A4Uht2xS3TCmlRgonRzDXGPOegMdfE5F9YbzvZ8D3gV+M8pqXjTE3hfFZF5U1ZVbC+I97q4ccr2jUiqVKqeknnB5Bt4hc6TwQkSuAMUtsGmO2AjE5dWZVaRYi1u/z81P55NVzADjbqGsLlFLTTzg9gr8Bfm7nBQTr5n7nBJ1/g4jsB2qALxhjDgd7kYjcBdwFUFJSMkGnjpz0RA+3rSriRF07P75jDXsqWwA4q4vMlFLTUDizhvYBy0Uk3X48UQPde4BSY0yHiNwIPAbMD9GGHwE/AlizZs1FUbvh27cv9/9elmtteq9DQ0qp6ShkIBCRDxtjHhaRfxh2HABjzL1v5sSBAcUY85SI/FBEco0xDW/mc6ej0mxr2ui55i68Az7i3FrrTyk1fYx2R3ImvacF+Ul9sycWkRliRxURWWe3JSpLVyTFu5mRnkj/gKFW9zpWSk0zIXsExpgH7F//Yox5NfA5O2E8KhH5DXANkCsiVcBXAY/92fcDtwF/IyJerOTz+00Ul+wszUnmfFsPZxs7Kc5OnurmKKWUXzjJ4u8Bw2sLBTs2hDHmA2M8/32s6aUxoSwnhe3lTZxt6GTj/Lypbo5SSvmNliPYAFwO5A3LE6QD7kg3LNqU2gnjs5owVkpNM6P1COKxcgFxWHkBRxvWsI4ah7IcK+VSoWsJlFLTzGg5gpeAl0TkZ8aYiklsU1RyAoH2CJRS00048xh/IiKZzgMRyRKRZyPXpOhUmmMNDVU2djHgi9qcuFLqIhROIMg1xrQ4D4wxzUB+xFoUpVIS4shLS6BvwEdt65gVOpRSatKEEwh8IuKv6yAipVgb06hxKsvRFcZKqeknnEDwZeAVEfmliPwS2ArcE9lmRafBPIEmjJVS00c4tYaesfcoXo9VdO7vo7EMxGQoy3VmDmmPQCk1fYSzoAwgAavqaBywWEScMtNqHJyEsVYhVUpNJ2MGAhH5JvA+4DDgsw8brCEiNQ6Dawm0R6CUmj7C6RHcAiwwxvRGuC1Rz98jaOzE5zO4XDLFLVJKqfCSxWewi8WpNyct0UNuajy9Xh917VqFVCk1PYTTI+gC9onIZsDfKzDGfDZirYpis3NTaOjo4/j5dmZmJE11c5RSKqxA8Lj9oybAqpIsdp5tZufZJq5ZEHpdXmt3Py8cq+PZQ3UcqW3jv969lCvm5U5iS5VSsSKc6aM/n4yGxIp1s7N5YOsZtp9pCvma5w6f59O/3kvfgM9/7IkDtRoIlFIREc6soXKCrCQ2xsyJSIui3JqybERgf1ULPf0DJHpGVvR+8mAtfQM+lhVlsHBGGr/bVUVVs840UkpFRjhDQ2sCfk8EbgeyI9Oc6JeR5GHRjHSO1Laxp7KZy+eO/JZ//Hw7AF971xKS4t38blcV1S1an0gpFRljzhoyxjQG/FQbY+4Dro1806LXZXOsOLqjfOTwUP+Aj9MXOgCYX5BGYaaVUK5u7iaKd/JUSk2hMQOBiKwK+FkjIp9i6EY1apwum50DEDRPUN7QSf+AoTg7idSEONISPWQkeej1+mjo6JvspiqlYkA4Q0PfCfjdC5QD741Mc2LDutlWj2BPZTO93gES4gbzBMfsYaEFBen+Y0VZSbR291PV3EVeWsLkNlYpFfVC9ghEZD2AMWZTwM9bjTF3GWOOT14To092SjyXFKTS6/VxsKp1yHPHz7cBsHDGYKerKMseHtI8gVIqAkYbGvqh84uIvD4JbYkpTq9g+7A8wfHzVn5gQUAgKMy0SlNUNWsgUEpNvNECQWAhnMRINyTW+PMEwwNBndUjWBCsR6CBQCkVAaPlCFwikoUVLJzf/cHBGBN6RZQa02V2j2D32Sa8Az7i3C46er2ca+rG4xZm23sXABTagUDXEiilImG0QJAB7Gbw5r8n4DkD6IKyNyE/PZHZuSmUN3RyoLqVVSVZnKizEsVz81LxuAc7a5ojUEpFUshAYIwpm8R2xKSr5udS3tDJY3urWVWS5V9IFpgoBijKGswRGGMQ0fLVSqmJE04ZahUh719XAsCje6rp7PX6A8GCGelDXpeR5CEtIY6uvgFauvonvZ1KqeimgWAKLZqZzurSLDp6vTy+vyYgEKSOeO1gnkCHh5RSE0sDwRT70GVWr+DhbRUcrwveI4DAPMH4EsaP7K7ipRMX3mQrlVLRLJwSE78M55h6Y25cOpPMZA+Ha9po6uwjLTGOWRkjZ+sG5gmCaezopc/rG3KsorGTL/x+P5/77V6tU6SUCimcHsGSwAci4gZWR6Y5sSfR4+a2VUX+xwsK0oImg53ic8ECweGaVjZ84wW++OiBIcf3VDYD0NzVz4UO3XJaKRXcaCUm7hGRdmCZiLTZP+1APfCnSWthDPigPTwEQxeSBSoaJUfw05fL6fP6eO5wHQO+wW/+eytb/L+fru+coNYqpaJNyEBgjPkvY0wa8G1jTLr9k2aMyTHG3DOJbYx6c/JSuWKetdJ48ayR+QEIHBoamiOob+/hzwdqAOjo9XK0ts3/3JBAYJe2Vkqp4cIZGnpCRFIAROTDInKviJRGuF0x55vvWcbn33oJ7wkYJgpUGGJR2a+3V9I/MNgLcEpW9PQPDAkKGgiUUqGEEwj+D+gSkeXAPwIVwC8i2qoYVJSVzGfeMj/o1pUAWckekuPdtPd4ae221hL0eX08vK0SgHevKgRgR3kjAIeqW/EGDBOdvqBDQ0qp4MIJBF5jTTm5GfhfY8z/ohvTTDoRGbJbGcBTB2tp6Ohl4Yw0PnvtfMDa9cwY4x8WWm/vhna6XnsESqngwgkE7SJyD/AR4El71pAnss1SwRQNKz730GtnAbjz8jJKc5IpSE+guaufU/Ud7D1nzRh61/JCPG6huqWbrj7vlLRbKTW9hRMI3gf0Ah8zxpwHCoFvj/UmEXlQROpF5FCI50VEvisip0TkgIisGlfLY5CTMH50TzV3PLiD/edayEz2cPOKQkSEdQGlrZ0ewbrZWZTmWJVMz+jwkFIqiHA2rz8P/ArIEJGbgB5jTDg5gp8B14/y/A3AfPvnLqxchBqFkzB+5vB5tp64gNsl/NP1C0mKt/IKzmY3j++voba1h7TEOObkpjI3zwoEmjBWSgUTzsri9wI7gNux9ireLiK3jfU+Y8xWYLQ9C24GfmEs24BMEZkZXrNj03WL8inMTOKKeTn8561L2fnl6/jAusE1CM4eBzvsmUMrijNxuYR5+VbtIs0TKKWCCWfz+i8Da40x9QAikgf8BXjkTZ67EDgX8LjKPlY7/IUichdWr4GSkpLhT8eMeflpvPrFa0M/n5dKVrKHZrtC6criTMDa3wB05pBSKrhwcgQuJwjYGsN831iCFdUPWhDHGPMjY8waY8yavLy8CTh1dHK5hLVl2f7HK0uygMBAoD0CpdRI4dzQnxGRZ0XkThG5E3gSeHoCzl0FFAc8LgJqJuBzY5qTJwBYbvcI5tg5gjMNnUNKUCilFISXLP7/gAeAZcBy4EfGmH+cgHM/Dtxhzx5aD7QaY0YMC6nx2TDXmjk0Lz+V7JR4ANISPcxIT6TP6/OvQRjNgM9Q39bDoepWTmleQamoFzJHICLzgAJjzKvGmEeBR+3jV4nIXGPM6dE+WER+A1wD5IpIFfBV7PUHxpj7gaeAG4FTQBfw0Td/OWrJrAwe+MhqSnOShxyfm5/C+bYeTl1op2TYc4Hufe44P3jx9JCew1Of3RiyBpJS6uI3WrL4PuBLQY532c+9c7QPNsZ8YIznDXD36M1Tb8Tbl8wYcWxuXiqvnmrkdH0n1y4M/d4/7KlmwGfITonHO+CjrcfLvnMtGgiUimKjDQ2VGWMODD9ojNkFlEWsRSoiwkkY9/QPUN3STZxL2P6lt3DXVXMAONuos42UimajBYKR22QNSprohqjI8q8lGCUQODf8kuxkPG4XZblWkrm8QQOBUtFstECwU0Q+MfygiHwc2B25JqlIcHoEp+o7Qm5bWW6vM3BmGZXZpSnOaiBQKqqNliP4HPBHEfkQgzf+NUA8cGuE26UmWEF6Apn2YrOa1h5/JdNAZ+wb/my7J+D0CCqauvD5DC5XsKUfSqmL3Wg7lNUZYy4HvgactX++ZozZYNcfUhcREWFZUSYA+8+1BH2NU5Rudq7Ve0hNiCMvLYE+r4+a1rGnnQ63t7KZzl6teKrUdBfOOoItxpjv2T8vTEajVGSsKMoAYH9VS9Dnyxus/IHTIwCY7R8e6gr6nlBePF7PrT98jW8+c+wNtFQpNZkmolSEukg4K41D9QicpLCTIwAoy7XWHJSPc+aQU/ju9dON42ylUmqyaSCIIc7Q0MGq1hGlJpo7+2ju6icl3k1+WoL/uJMnGG/C+Nj5dsCapaQb4ig1vWkgiCF5aQkUZibR2TcwYhqp841/dl4KIoNJ4dlvcObQsdo2AHwGjtq/K6WmJw0EMWZ5sZUn2DdseGh4otjhX0sQYmioraefB146TWt3v/9YS1cfNa09/seHqjUQKDWdaSCIMctDzBwKliiGwbUE55q68A74Rnzevc+d4L+ePsYPXzzlP+YMCzkOVbe+2WYrpSJIA0GM8SeMh80c8ieKhwWCpHg3M9IT6R8w1LT0DHluwGd46qBVMDYwKewMCzmrmQ/VaI9AqelMA0GMWVqYgUvgWG07Pf0D/uODQ0MpI94TaubQzrNN1Lf3Ata3/rYea3jI6RHcurIQl8DJuqHnUkpNLxoIYkxKQhzz89Pw+gxHnISuz/jrDM3OGxkIZoeYOfTkgcHtI3wGdtpTRo/agWBVSRZz81Lx+gwn6oYOFwVzsKqV4+fHfp1SamJpIIhBTsLYyRPUtvXQ0+8jNzWB9ETPiNc7eYLA4nMDPsPTh6xAsGmBtX3otjONDPgMx89bAWbRzDQuLbTONVbCuLPXy+0PvMYN/7uVbzx9jF6v9iCUmiwaCGLQ8FIT/mJzQYaFIGAtQcDQ0PbyRho6+ijNSebjV1rlql8/00hFYyc9/T5mZiSSmRzPEnsfg4NjJIzP2u/zGbj/pdO863uvckRzC0pNCg0EMWiFnTDed64FY0zIGUOOYENDzrDQO5bOZHVpFh63cLimzb+ieOGMNAB/j+BwzeiBoKKxy/++2bkpHK9r544Hd9AfZKaSUmpiaSCIQQtmpJEc7+ZsYxef/vVe9lS2AMHzA2DtTyAC55q76R/w4R3w8cwhq+7gO5bNJCnezYriTIyBn79eAcDCmVZPwNnZ7Fht+6g3dScQXDEvlyc/eyUzMxJp6OilQjfFUSriNBDEII/bxbduW0ZKvJsnD9byx73VQOgeQaLHzayMJAZ8htdPN/Kr7ZU0dvYxOzeFxfYNf8OcHGBwFbHTI0hP9DA7N4W+AR8n60JviuPc8MtykkmOj/N/7mjvCaW7b4DdFU0h911QSg2lgSBG3bRsFs987irWzc72H3M2rwnGmUJ6x4M7+OrjhwG4cekMfzmK9XYgcDg3csCfJzg0yvCQ0yMosRPTzhqEk/XjDwT3bT7Be/7vdZ48WDv2i5VSGghiWXF2Mr/9xHr+45ZL+Ye3XsLcEENDAO9YOovkeDfF2UlcMS+Hv9pQyl/bSWKAVaVZxLutv07xbteQ3oU/TzBKwjiwRwBvLhBsP2PlKV44Wj/u9yoVi0bboUzFAJdL+PD60jFf98HLSvjgZSUhn0/0uFlRksmO8ibmF6QS5x78jnHpLGcfhOCBoKd/gNq2HuJc4t85bX6BNbR0Moz1B4Gs6avWe7aXW8NDgUX0lFIjaY9ATRhneGhRwLAQwFJ7Q5wjtW30eUcmjKuauzAGCrOS/AHE6RGcaegMWuMolIrGTrrtVczVLd1UNY9/ZzWlYo0GAjVhPnZFGXdeXsbdm+YNOZ6R5GFuXgp9Xh/Hzo9cG+DsflaaMziclJoQx6yMRPq8Ps6N42Z+tHZoD2LbmbE3xmnq7OO9D7zOI7urwj6PUtFEA4GaMJnJ8fzru5YEnX20PGDtwnBnh+UHHPPGGB76r6ePsvbrf6E2YD9lZ9ZSVrK1QnqbnS8YzQvH6tlR3sQ9jx4IGqiUinYaCNSkWDlKIKhssmcMZQ8NBPNHSRj3D/j49bZKLrT3+tc0wGAgeP86K5+xvXzsHkGlHYj6Bwyf/93+oMNXSkUzDQRqUoy2X/JZe+poWc7QnoQTCE4FCQS7zjbT3mttgfnqqQb/cafy6btXFpKeGEdVczdVzV2jts0JRCJwuKaN7285NerrlYo2GgjUpFg4I534OBenL3QO2c0MAqaO5g7rERQ4PYKRQ0MvHh+cGrrtTBP9Az5au/qpbukm0eNiTl6qf43E9jGGhyrsQPD3112CCPxgyykODNuvQalopoFATYr4ONdgAbqAaaT9Az6qm7sRgaKsYTmCPCtHcLq+E59v6CrhLXYgiHe76Oj1sv9cC0ft8f0FBWm4XeKfxTRWwvicHQhuX1PEx66YzYDP8O1nj7+h63x0TxX/9dRRBny6qlldPDQQqEnjbJO571yz/1hNSzden2FmeiKJHveQ12cke8hPS6C7f4DqlsGEcFVzFyfqOkhNiOM9q4sAePlkgz8/4ExfdQLB9vLQPYKOXi8NHX3Ex7koSEvkrqvm2G1sGXeJCmMMX/vzER7YeobtYcxWAmsNxc6zWg5DTS0NBGrSrCzJBGDfucEegZMfKM0JvqrZGR4KzBO8ePwCAFfOy/XvhfDqqZGBYNHMdNIS46hs6qKmJfgUVKc3UJyVhMsl5KclkJ0ST3uPd0jwCce5pm7/sNdzR+rCes+Ptp7h9vtf56evlI/rXEpNJA0EatIM9ggGv207M3ZKh00ddczPt6eQBuQJnPzApoV5rJ+bg9sl7D3Xwq4Kq6fhBAK3S1hXZuUJQg0P+Wsc2TOWRIRFM61zDl+TMJbAPReeP1IX1rd85z0/2npGN+NRU0YDgZo0pTnJZCZ7aOjopaa1Bxi7RzDXmUJqVyHt6R/g1VPWTf2aBfmkJ3pYXpTBgM/4911eaN/IAdbaCePdFc0E4/QIAs/vFMxzehjhCgwE1S3d/hlMo3HOX9/ey5/21ozrfEpNFA0EatKIyGCvwN4DYXixueGGryXYUd5Ed/8AS2alU5CeCMCV8/P8ry/KShqy3eaqkiwA/54Lw1U0WecvDljDsCiMQPDs4fNsPjp0+OeQHQhm2O16fozhIWOMf+oqwANbT49Iiis1GTQQqEnlX09gT88cLD89eiA4UdfO9184yY+2ngFg04J8/2uunJfr/314naNlRRnEuYTj59vosNcdBKpssvIApeMIBI0dvfzNw7v55C9309LVB1g3dafM9t2b5gJjB4Kmzj66+gZIs8tpnL7QyeZjk1cx9fj5dp49fH7sF6qop4FATSpnhfFPXynnfQ+87g8EoYaGclITKEhPoKtvgP9+7gSv2IvHNi0cDAQrSzJJibdmHA0PBIkeN4tnpeMzcCDYqma7RxIYiObmpeJxC2cbu4IGj1dONeAz4PUZXjphJa6rmrtp6eonOyWe29cUk+Rxc7C6dUj5ixHndoalcpP5+EZrttIDL50O+frRGGP43uaT/Glfddjv+cLv9/PJX+5m59mxy3Co6BbRQCAi14vIcRE5JSJfDPL8NSLSKiL77J9/iWR71NTbMDeHa+yZPtvLm+gb8JGXlkBqQuiK6Pd/eDWfu24+d2+ayyc2zubfbl7CKnsGElg7rm20h4cCjzuc4aG9wwLBgM/4q5MWB6xhiI9zMc9OUh8PUnvo5ZODK5k323seOMNClxZmkOhxs3G+1Uv5yyh7IgSW1nj/2mIykjzsqmhmd8X4b8xHatv4zvMn+MdHDtDW0z/m640x/gT8o3u02F6si9h+BCLiBn4AvBWoAnaKyOPGmCPDXvqyMeamSLVDTS+JHjc/++g6Wrv7efnkBV4/3ThkaCeYlSVZrLRv5qF8/dZLuW11EVdfkjfiuZUlmfzsNdgzLGHsrGHIT0sgKX7oGoZFM9M4WtvGkdp2VpcO7uJmjOHlkxf8j188Xo93wOdPFC8ttHokb11cwHNH6nj+SB0fCbHfg3/qanYyKQlxfGBdCfe/dJo/768dcs5w7LVzIL1eH08eqOUD60LvHQFwoaOXnn6rptITB2r56juXjFjHoWJHJHsE64BTxpgzxpg+4LfAzRE8n7qIZCR5uGnZLL5+61JuWDrzTX9eTmoC1y0uCLoJTWCPIHBK5+CMoZH5iVAzh07UdVDX1kteWgJzclNo6/Gyu6I5IBBYey9cuzAfl8DrpxtoD/ENfXixvcvnWgvg3kh5iz2Vg0HuD2GU0z4XkKRu7/HywiTmJgAefKV8XMNYKrIiGQgKgXMBj6vsY8NtEJH9IvK0iCwJ9kEicpeI7BKRXRcuXAj2EqVCKspKIjc1gabOPv90VRisMVScPTIQhEoYO72BjfNzecsiK0+x+Vi9f2hoib0bW05qAitLsugfMOwIsbJ5eCBYZm/gc7imjf5xbMYDg7OwRGBXRbN/NlYozrnjXFbgfHTP5N2Uz7f28G9PHOELv98fMkiqyRXJQBBsf8Dhc+P2AKXGmOXA94DHgn2QMeZHxpg1xpg1eXkju/5KjUZE/LmDvQHfnP3J2uyRiWonEBw/3z5kSudWOz9w1fw83rKoAIBHdlfR3NVPZrKHoqwk/2udone7Qq5hsPITTiDITI6nLCeZXq/Pv93mcD39A/zzYwf9SWqA5s4+zjR0kuhx8c5lswD4wxg3dufct6wsxO0SXjxeT1Nn36jvGU1dW0/YU1/PBpT9DrwONXUiGQiqgOKAx0XAkBUzxpg2Y0yH/ftTgEdERh8wVuoNWOlfTxAQCPxTV5NGvD47JZ4Z6Yl09Q34ew49/QP+GkJXzs9ldWkW6Ylx/hvo0sKMIUNTa0qtc+4+OzIQ9Hl91LR24xKYlTl4fmd67YEQ+zs/caCWh7dV8s+PHfQPczl7PCwrzOR9a61/co/uqRr1xuwEwVUlWVw5Lxevz/DkgTe2oO21Uw1c9p+b+VaYhfoC106MNcV2ovl8hoe3VYy7fEi0i2Qg2AnMF5HZIhIPvB94PPAFIjJD7H85IrLObk941bqUGgenR7CnosV/zFlMVhKkRwAElJqwhod2nm2i1+tjyax0clMT8LhdXB2wnuFSOz/gWG0Hgn1VLSPKR1S3dGMMzMxIwuMe/Ge4zF5wFypP4AxNnWvqZr8dLJxezsqSTNbPyWFmRiJVzd3sGGVaaOCw1LtXWSO2j+59Y8NDT9sbA/38tbP+dRWjqQoIBC8cqx/3MNib8dyR8/zzY4f4ymOHJu2cF4OIBQJjjBf4NPAscBT4nTHmsIh8SkQ+Zb/sNuCQiOwHvgu832gZRhUBy4oyiXMJx8630WmvDagcVmdouOF5Amfa6MaAlcxvCVjPsHRYIMhMjmd+fip9Xh+HqofmGkLtyrai2PqMYDu5+XxmyNTVJ/Zb3+CdVdMrSzJxu4RbV1o39tGSxlUB53/b4hmkxLvZW9nC2YbRcwvBOHWcuvsH+O3Oc2O8emiPoL3HO+Z+ERPJKV742ukGevq1tpMjousIjDFPGWMuMcbMNcZ83T52vzHmfvv37xtjlhhjlhtj1htjXotke1TsSop3s2imvbCsqpWWrj7aerwkx7vJTY0P+h4nEDx5oJZ7nzvO04dqAbhq/uDo5TUL8rDzrSMCAcCaMnt4aNjagFCBYPHMDNwu4WR9B119QxezHa5po6mzj4Q465/tkwdr8Q74/EHDGf5ySnM/c+g83iDftnu9A9S29eASmJmZSFK8m2vsgPZKwG5v4Wjs6B2ylejPXzs75jd859qdHtPzRyZvdXOlf5jPN+kL6XaUN/Hhn2wfc8e8qaAri1XMcMpgf+mPB/nG08cA60YcbMopwIriTETgTEMn333hFOeauknyuFldNrimITM5nn991xL+4a2XBJ195KwH2DksT+BM3xxeWiMp3s0lBWkM+AyHa4b2Irbaw0K3riykMDOJ2tYe/t+uc3T0einMTPLXXpqbl0ppTjLtvd6gFVSrm61hqVmZg8NSl8122hn65tje089vdlQO+SbtzIjaMCeHuXkp1Lb2+IeKQjlnL+L7+JWzgfArtQZzqLqVf/nTIZrDTHQH9kZeOj65ieqHt1XwyqkGfrmtYlLPG46ILShTarq5ZWUhj++vobyhk/IGJz8QfFgIrGmlj3xqA4dr2mjo6KOps5eN8/NIiBu68OqODWUhP2OtHTT2VDRjjPEHHWdYKljwWFGcwdHaNvafa2Ft2eDCsq32DJurLskjI8nDA1vPcO9zJ6z3DFtRva4sm4rGLraXN7K0aGhPxbkRB167c56do2zi893NJ/nxy+WUN3TypRsXAYOb/qyfk0NuWjxf/uMhHnylnHctnxX0M7r7BrjQ3ovHLbxtcQH5aQnUtPZwuKZtRI4lHN985hgvn2zA6zP8561Lx3y9M1sKBgPrZHEmHbx+evqlQbVHoGLGqpIsdnzpOh766FpuX13E3LwU3r2qaNT3rC7N5o4NZfzDWy/hP25ZytuXzBjXOUuyk8lNTaCxs88ffCD00BAMJoz3B8wc6ui1Fq65BK6Ym8tN9jTRRvubsFPDyeFMXQ22hqHSvxnP4LkXFKSRnhhHTWtPyKELp5zG73ed8ye/nfzA+jnZvHtlEZnJHvadawld9tv+7KKsZOLcLq5bbE3BDXcjn0BdfYP5hd/tPDdmfiNwplZKvJsTdR2j1oKaaE5dq4PVrbR2Ta/1ExoIVEyJj3OxaUE+3759OZs/fw3XXzq+G/t4iYh/Gukue3jIGDM4NBQkECwPMnNo2+lGvD7DiuJMMpI9XFqYPqR096rSoSU4LpttrVLeebZpxDTSYMNSLpcM9gqCDA9VNnZxxr7RNnf18+zhOpo7+zh2vp34OBfLizNJinfzQbu0xcMhhj/ODVvE91YnELyBKqjbz1i1qsAqAHjfX06M+npnptaszCQ2zLXyPFsnaR1DW08/zfbN3xjYVj69egUaCJSKMCdhvMtOGLd09dPe6yU1IY6sZM+I119SkEqix0VFY5d/OuZW/4pma8aSiPh7BfFuF0tmDa26WpydxIz0RJq7+jl1oWPIc8Nvxo61/l7EyG/zL56wegNOldffbK/0T09dVZLpr1Pk9LBePtkQdNx/sDdirZ24fG4OyfFujp1v57y9WVEwzZ19I1Z5O4vR3r2yEI9b+NP+mpAL8WBw74uS7GSuvsQJBONLjgf63uaTfPrXe8KafVTZOLSX9do4k/KRpoFAqQhbUzZ0hbEzPFIcIlEd53b5S1U4w0OB+QHHLSsLiY9zcfm8nBF5CxHxDw9tLw8+Y6k4a+hCutF6BM4+0Z9/2wKSPG5eP9PIb3dUAoO9D4C5eSnkpSXQ0NHL6WEBKPDcTk8oIc7NhjnW+18eZcz+07/Zw43ffXlI25xA8KH1JXxwXQnGwHeeC72oLbAXdvUl+f5zBptZNZbmzj7u23ySJw7U8oMtp8Z8vXNuZ4baq9MsT6CBQKkIWzIrnUSPizMXOqlq7gq4GY5c0exwhof+6ZEDfPDH2zjb2EV6YhzLAxK/8/JT2fwPV/O9D6wM+hmh8gSh8hNLCzNI9Lg4Vd8xpNxET/8Ar522vsHetGwm71xuFQncYgeHy+YMJrRFhPX2jT1YUnR4WQ3AX7J768ng35IbO3p57XQjxuC/6VY0Wgl/688kk7uvnUeix8VzR+rYH2QNRuB1F2cnU5KTTFlOMm093iG5mHA9f7SOAXvI7f6XTnOibvRtSZ1E8Q2XziQ53s2p+g7q20L3gCabBgKlIszjdvm/bb/lOy/x3c0ngdFnLL11cQEet3C+rYfXTg/u0RznHvpPtjg7mbTEkcNLMDgldEd5o3+YprWrn/YeLynxbrJThq6fiI9zscJOOgd+895R3kRPv4/FM9PJT08cUuI63u3yV3d1ON/wtwVZKBZsWGqj3ct55eSFoGUxthy/gDPK9OLxCxypafP3BjbOzyPO7SI/LZEPXWaV+/7TvuClMoYHQKd39UbqHT1jT5EtzEyif8Bwz6MHwyrpMTcvxf934bVp1CvQQKDUJPj6LUvZtCCPXq+PE3XWkEmwqaOODXNz2POVt/L0323koTvX8p3bl/PVdy4e1znn5aeSnRJPXVuv/0YU+K042LDUuiDTSJ1hIWdDoRXFmSyckeb/ffg+BuvtHsK2M41D8gSBezQHXvuc3BQKM5No7uofsXYC4IVj1oyigvQEwPoG7qwBCNx/wqkG6/Rehqsc1htx3vvi8dAluH0+wyd/uYvP/mav/0bf1tPPKycbEIFffnwdeWkJ7K5o5tf2UFnQcwdsyXrFvJxR2zkVNBAoNQlKcpJ56KPr2Pz5q/nw+hLWz8kec8ZSWqKHRTPT2bQwn/esLiInNWFc5xQR/zoGJ08Q7EYcaG2QhWVOovgau66SiPAJe2vNYNcwOzeFgnRrymzgquOGjj66+wfISPKQkTTYixERrnKSt8PyBH1enz+h+/0PriLOJTxxoMa/AjowZ7KqJIuEOBfHzrfT0NE75HOCzdS6Yl4uSR43B6paqQlRhG7vuRaePVzH4/treOKgtbL8haP19A34WFuWzZy8VL72Lqt6/jefPhay1lJgXavL7RlLr55qfMML6SaaBgKlJtHcvFT+45al/PauDeSnJUb8fOvsRK6TJ3AS1aGGpVaVZOF2CYdq2ujo9XKuqYszFzpJS4wbsg3oe1YX8co/beLOy8tGfEaoPMFgknxkbsSZDTU8YbzzbBMdvV4WFKSxtiybW1YW4jPWTmwLZ6QxI2PwzzDR4w457NLc1U9Hr5e0hDgy7ZlaiR63PwCFqoIaePxbzxyj1zvgLzVygx0Eb7h0ButmZ9Pe6w06zNQ/4KOmpQcRa2+MxTPTyUjyUN3SPWSBWzD9A75JCRYaCJSKYk6e4NVTDdS19YScMeRISYhjyax0BnyGVf/+PO/5P6v818b5uSPyE0VZybhcwctzDOYJAgLBKGsnLp+bg0tgd0WzvyggDC5iu9Ye9vnU1XNwRrSuXjByb5LL5zkBaOiwS6ghMWeB4HMh6h05x9MT46hq7ub+F8/4b/ZOb0hEuG6RMwtp5HBPTUs3Az7DjPREEj1uXC4ZnCl1KnR+4lR9O0v/9Vm+8qfIV0rVQKBUFFs0M51ZGYnUtvZw1be28Kyd5Bxe4yjQx6+cTX5aAn1eH/Xt1hDL9ZeObzvR9QGBwBlbH62sRmZyPMuKMukfMGy3F1sZY9hs5wecKq/z8tO4efksXALvCLLF6RUBwy6BQs2UunZhPm6XsO1M04hhnVP1HZy50ElGkod737sCgP/5ywl6+n2sLMlkZsZgMA3s0Qz/Bh9sOG7TQuv1Tx6oHXENjt/vqqKn38fD2yrZEuGtRDUQKBXF3C7hV59Yz41LZ9Dr9flLUow2Y+nmFYXs+PJ1HP7a23nqsxv5w99s4J3LxhcISnOSmZlhLWg7UW9NrfQPDWUFP7dT1dXJCZxp6KSisYvMZI+/sirAt25bziv/dK2/FEegSwszSE+Mo7Kpa8i+zKGK/GUmx7N+TjYDPjNi32anN/CWRfm8ZVE+V84brDp7w7DcyMIZaeSmJlDX1usvde2oaHR2whs89/WXziQ+zsXrZxqDlrkwxvDUocEg8aU/Hozotp4aCJSKcrNzU/jhh1bzxGeu5O1LCti0II/Zualjvi8lIY7Fs9JZXZodskJrKMHyBKPVV4LBaaTPH6njL0fq/PspbFpgfWt3xMe5huzqFsjtGjxv4Kyc0XojzvDQs8PKXDj5gbctLkBEuOfGhf5hqeuXDA2MIsKV85yFcUOHh5wgVBoQhDKSPFy3KB9j4PEg010P17RxrqmbvLQElhdlUNvawzefORb0mieCBgKlYsSlhRk88JE1PPTRdUNurJHijIP/ansl+861BF1MFmhFcaY/ifrXv9jFD188DVjDN+NxxbyRw0OjBSGn3tFLJy7Q3WeVi6hv62FvZQsJcS7/zKQlszK4730r+NZty4IOrYVKeFeECEK3rLA2EPpjkJ3hnIT09Utm8M3bluFxCw9vqxySc5lIGgiUUhFx3eICCtITOFXfwS0/eJXqlpF7NAfyuF08eOda/vrK2Vw5L5eclHjm5af61y+Ea3Ce/uD0zNECwcyMJJYXZdDT7/PfxJ8/avUGNs7PJTl+sFr/zSsKee+a4hGfAdY+1mBN1e3zDpatqPD3CIZuiXrNgnwykz0cO98+pI6SMYanD1q9kxsuncHCGen87TXzAPjiHw5EZGc13Y9AKRUR2SnxPPf3V3P/S6d58JVyer0+ZmUmER8X+vvn6tIs/85lb9TcvFTy0xKob+/lRF0Hs3NTqLXLTxeGCEJvWzKD/VWtfH/LKRo7+/jTXmu45m2Lw69OW5CeyIKCNI7XtbOnspn1c3JGrTQbH+fiHUtn8qvtlTy2t9q/I96Jug7ONHSSnRLvLxNy96Z5bC9v5APrSvw71E0k7REopSImI8nDP12/kC1fuIZPbJzNl+0NbSJJRPzDQ19/6iivnm7AZ6xv/qGC0I1LZ+J2CQeqWrnn0YPsONuEyOC01XA5vQKnZ9HU2edfvxCs0uy7V1nDQ4/tq/bXLnrKXrj2tsUF/im78XEufvOJ9dy8onDc+ZpwaCBQSkXcrMwkvvyOxdwQZMpnJHx4vfXNeeuJC3z0oZ3A6DOlZuem8MzfbeRfblrMTctmUpqTzB3rS8kd52puJxC8YieMxyrpsaoki+LsJOraevn1jkq6+rz+OkbD/6wiEQAcOjSklIo6q0uz2fKFa7jvLyd4ZHcVPgNluaEDAcD8gjTmF6TxMWa/4fNeNjubeLeLA9WtnKpv9weC0hDrNkSEW1cW8d3NJ/nKY4f4tz8fpn/AkJ4Y50+2TwYNBEqpqDQrM4lv3bacT2ycwx/3VvO+tcGTvBMpOT6OK+fn8sKxet5+38v+FdyjLeC766o5YAwvnbjAgWqrJPZNy2eNmkuZaDJdih6Fa82aNWbXrl1T3QyllArqQnsv//3scR7ZU+Uf9//6rZf6y2SPpqWrjyO1bawozhwyW2kiiMhuY8yaoM9pIFBKqYlX2djF97ec5GR9Bw98ZPWkFBkczWiBQIeGlFIqAkpykvnWbcunuhlh0VlDSikV4zQQKKVUjNNAoJRSMU4DgVJKxTgNBEopFeM0ECilVIzTQKCUUjFOA4FSSsW4i25lsYhcACre4NtzgYYxXxV9YvG6Y/GaITavOxavGcZ/3aXGmKC7/Fx0geDNEJFdoZZYR7NYvO5YvGaIzeuOxWuGib1uHRpSSqkYp4FAKaViXKwFgh9NdQOmSCxedyxeM8TmdcfiNcMEXndM5QiUUkqNFGs9AqWUUsNoIFBKqRgXM4FARK4XkeMickpEvjjV7YkEESkWkS0iclREDovI39nHs0XkeRE5af83a6rbOtFExC0ie0XkCftxLFxzpog8IiLH7P/nG2Lkuv/e/vt9SER+IyKJ0XbdIvKgiNSLyKGAYyGvUUTuse9tx0Xk7eM9X0wEAhFxAz8AbgAWAx8QkcVT26qI8AKfN8YsAtYDd9vX+UVgszFmPrDZfhxt/g44GvA4Fq75f4FnjDELgeVY1x/V1y0ihcBngTXGmEsBN/B+ou+6fwZcP+xY0Gu0/42/H1hiv+eH9j0vbDERCIB1wCljzBljTB/wW+DmKW7ThDPG1Bpj9ti/t2PdGAqxrvXn9st+DtwyJQ2MEBEpAt4B/CTgcLRfczpwFfBTAGNMnzGmhSi/blsckCQicUAyUEOUXbcxZivQNOxwqGu8GfitMabXGFMOnMK654UtVgJBIXAu4HGVfSxqiUgZsBLYDhQYY2rBChZA/hQ2LRLuA/4R8AUci/ZrngNcAB6yh8R+IiIpRPl1G2Oqgf8GKoFaoNUY8xxRft22UNf4pu9vsRIIJMixqJ03KyKpwB+Azxlj2qa6PZEkIjcB9caY3VPdlkkWB6wC/s8YsxLo5OIfDhmTPS5+MzAbmAWkiMiHp7ZVU+5N399iJRBUAcUBj4uwupNRR0Q8WEHgV8aYR+3DdSIy035+JlA/Ve2LgCuAd4nIWawhv2tF5GGi+5rB+jtdZYzZbj9+BCswRPt1XweUG2MuGGP6gUeBy4n+64bQ1/im72+xEgh2AvNFZLaIxGMlVh6f4jZNOBERrDHjo8aYewOeehz4K/v3vwL+NNltixRjzD3GmCJjTBnW/9cXjDEfJoqvGcAYcx44JyIL7ENvAY4Q5deNNSS0XkSS7b/vb8HKhUX7dUPoa3wceL+IJIjIbGA+sGNcn2yMiYkf4EbgBHAa+PJUtydC13glVpfwALDP/rkRyMGaZXDS/m/2VLc1Qtd/DfCE/XvUXzOwAthl//9+DMiKkev+GnAMOAT8EkiItusGfoOVA+nH+sb/8dGuEfiyfW87Dtww3vNpiQmllIpxsTI0pJRSKgQNBEopFeM0ECilVIzTQKCUUjFOA4FSSsU4DQRqyomIEZHvBDz+goj86wR99s9E5LaJ+KwxznO7XQF0y7Djs0TkEfv3FSJy4wSeM1NE/jbYuZQaDw0EajroBd4tIrlT3ZBA46zg+HHgb40xmwIPGmNqjDFOIFqBta5jPG2IG+XpTMAfCIadS6mwaSBQ04EXa//Vvx/+xPBv9CLSYf/3GhF5SUR+JyInROQbIvIhEdkhIgdFZG7Ax1wnIi/br7vJfr9bRL4tIjtF5ICIfDLgc7eIyK+Bg0Ha8wH78w+JyDftY/+CtZjvfhH59rDXl9mvjQf+DXifiOwTkfeJSIpdd36nXTjuZvs9d4rI70Xkz8BzIpIqIptFZI99bqdy7jeAufbnfds5l/0ZiSLykP36vSKyKeCzHxWRZ+y69t8K+PP4md3WgyIy4v+Fil6jfdtQajL9ADjg3JjCtBxYhFWu9wzwE2PMOrE25PkM8Dn7dWXA1cBcYIuIzAPuwKpcuVZEEoBXReQ5+/XrgEuNVdLXT0RmAd8EVgPNWDfpW4wx/yYi1wJfMMbsCtZQY0yfHTDWGGM+bX/ef2KVxPiYiGQCO0TkL/ZbNgDLjDFNdq/gVmNMm91r2iYij2MVmbvUGLPC/ryygFPebZ93qYgstNt6if3cCqzKtL3AcRH5HlYly0Jj1fjHbo+KEdojUNOCsaqk/gJr05Fw7TTWHgy9WMvrnRv5Qaybv+N3xhifMeYkVsBYCLwNuENE9mGV6s7BqtECsGN4ELCtBV40VsEzL/ArrD0B3qi3AV+02/AikAiU2M89b4xx6tEL8J8icgD4C1aJ4YIxPvtKrPILGGOOARWAEwg2G2NajTE9WPWJSrH+XOaIyPdE5HogqqvWqqG0R6Cmk/uAPcBDAce82F9Y7CJj8QHP9Qb87gt47GPo3+3hdVQM1s31M8aYZwOfEJFrsEo6BxOs3O+bIcB7jDHHh7XhsmFt+BCQB6w2xvSLVWk1MYzPDiXwz20AiDPGNIvIcuDtWL2J9wIfC+sq1EVPewRq2rC/Af8OK/HqOIs1FANWHXrPG/jo20XEZecN5mAV5noW+BuxynYjIpeItbHLaLYDV4tIrp1I/gDw0jja0Q6kBTx+FviMHeAQkZUh3peBtedCvz3WXxri8wJtxQog2ENCJVjXHZQ95OQyxvwB+ApWSWsVIzQQqOnmO0Dg7KEfY918dwDDvymH6zjWDftp4FP2kMhPsIZF9tgJ1gcYo4dsrF2h7gG2APuBPcaY8ZQ73gIsdpLFwL9jBbYDdhv+PcT7fgWsEZFdWDf3Y3Z7GrFyG4eGJ6mBHwJuETkI/D/gTnsILZRC4EV7mOpn9nWqGKHVR5VSKsZpj0AppWKcBgKllIpxGgiUUirGaSBQSqkYp4FAKaVinAYCpZSKcRoIlFIqxv3/XAZUWAgGHEgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "W1_t,b1_t,theta3_t,bias3_t,cost_t,accuracy_t = main_init(t,W1,b1,theta3,bias3)\n",
    "plt.plot(cost_t, linewidth = 2)\n",
    "plt.ylabel('Cost Function')\n",
    "plt.xlabel('Number of iterations')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}