Disaster_texts_prediction_NLP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMH9vBnHshe39GoENoMpFf8",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/thayyilakul/Tensorflow-ML/blob/main/Disaster_texts_prediction_NLP.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SQiQS4l3PdLq",
        "outputId": "7c48fa5f-205f-4f0d-9b95-0e4fe884107b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.17.1\n"
          ]
        }
      ],
      "source": [
        "import tensorflow as tf\n",
        "print(tf.__version__)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://raw.githubusercontent.com/mrdbourke/tensorflow-deep-learning/main/extras/helper_functions.py"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V2ewWXNzjIfy",
        "outputId": "0b37018e-f28d-4b45-da09-19fcf062d6f5"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-12-18 14:00:43--  https://raw.githubusercontent.com/mrdbourke/tensorflow-deep-learning/main/extras/helper_functions.py\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 10246 (10K) [text/plain]\n",
            "Saving to: ‘helper_functions.py’\n",
            "\n",
            "helper_functions.py 100%[===================>]  10.01K  --.-KB/s    in 0s      \n",
            "\n",
            "2024-12-18 14:00:43 (66.1 MB/s) - ‘helper_functions.py’ saved [10246/10246]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from helper_functions import unzip_data, create_tensorboard_callback, plot_loss_curves, compare_historys"
      ],
      "metadata": {
        "id": "w7-joxYvjjeI"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!wget \"https://storage.googleapis.com/ztm_tf_course/nlp_getting_started.zip\"\n",
        "unzip_data('nlp_getting_started.zip')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "erRUuy7ajV2V",
        "outputId": "12d6634e-9a51-4f45-fe52-df65e6eaad44"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-12-18 14:00:48--  https://storage.googleapis.com/ztm_tf_course/nlp_getting_started.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 74.125.132.207, 142.251.183.207, 74.125.201.207, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|74.125.132.207|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 607343 (593K) [application/zip]\n",
            "Saving to: ‘nlp_getting_started.zip’\n",
            "\n",
            "\rnlp_getting_started   0%[                    ]       0  --.-KB/s               \rnlp_getting_started 100%[===================>] 593.11K  --.-KB/s    in 0.007s  \n",
            "\n",
            "2024-12-18 14:00:48 (81.9 MB/s) - ‘nlp_getting_started.zip’ saved [607343/607343]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "train_df = pd.read_csv('train.csv')\n",
        "test_df = pd.read_csv('test.csv')\n",
        "train_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "4EawIRgYjsXI",
        "outputId": "13d5437d-4e52-4edc-aa5d-c92d01e6d759"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   id keyword location                                               text  \\\n",
              "0   1     NaN      NaN  Our Deeds are the Reason of this #earthquake M...   \n",
              "1   4     NaN      NaN             Forest fire near La Ronge Sask. Canada   \n",
              "2   5     NaN      NaN  All residents asked to 'shelter in place' are ...   \n",
              "3   6     NaN      NaN  13,000 people receive #wildfires evacuation or...   \n",
              "4   7     NaN      NaN  Just got sent this photo from Ruby #Alaska as ...   \n",
              "\n",
              "   target  \n",
              "0       1  \n",
              "1       1  \n",
              "2       1  \n",
              "3       1  \n",
              "4       1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-919b7326-91d9-41a7-a59f-8865d8e90ccb\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>All residents asked to 'shelter in place' are ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>6</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>7</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-919b7326-91d9-41a7-a59f-8865d8e90ccb')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-919b7326-91d9-41a7-a59f-8865d8e90ccb button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-919b7326-91d9-41a7-a59f-8865d8e90ccb');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-33fee7d3-d3e4-4258-9d69-074708001331\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-33fee7d3-d3e4-4258-9d69-074708001331')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-33fee7d3-d3e4-4258-9d69-074708001331 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "train_df",
              "summary": "{\n  \"name\": \"train_df\",\n  \"rows\": 7613,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3137,\n        \"min\": 1,\n        \"max\": 10873,\n        \"num_unique_values\": 7613,\n        \"samples\": [\n          3796,\n          3185,\n          7769\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"keyword\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 221,\n        \"samples\": [\n          \"injury\",\n          \"nuclear%20reactor\",\n          \"engulfed\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"location\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3341,\n        \"samples\": [\n          \"Oklahoma\",\n          \"Starling City\",\n          \"Trinidad and Tobago\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 7503,\n        \"samples\": [\n          \"Three Homes Demolished in Unrecognized Arab Village - International Middle East Media Center http://t.co/ik8m4Yi9T4\",\n          \"Reid Lake fire prompts campground evacuation order http://t.co/jBODKM6rBU\",\n          \"FAAN orders evacuation of abandoned aircraft at MMA http://t.co/dEvYbnVXGQ via @todayng\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"target\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_df_shuffled = train_df.sample(frac = 1, random_state = 42)\n",
        "train_df_shuffled.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "hdJA6aolj6Dn",
        "outputId": "95a203c0-5fb2-4bd4-9ecc-35b0ab2340c4"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "        id      keyword               location  \\\n",
              "2644  3796  destruction                    NaN   \n",
              "2227  3185       deluge                    NaN   \n",
              "5448  7769       police                     UK   \n",
              "132    191   aftershock                    NaN   \n",
              "6845  9810       trauma  Montgomery County, MD   \n",
              "\n",
              "                                                   text  target  \n",
              "2644  So you have a new weapon that can cause un-ima...       1  \n",
              "2227  The f$&amp;@ing things I do for #GISHWHES Just...       0  \n",
              "5448  DT @georgegalloway: RT @Galloway4Mayor: ÛÏThe...       1  \n",
              "132   Aftershock back to school kick off was great. ...       0  \n",
              "6845  in response to trauma Children of Addicts deve...       0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ca387025-47ed-4ae4-abbd-13adc44b25ac\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2644</th>\n",
              "      <td>3796</td>\n",
              "      <td>destruction</td>\n",
              "      <td>NaN</td>\n",
              "      <td>So you have a new weapon that can cause un-ima...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2227</th>\n",
              "      <td>3185</td>\n",
              "      <td>deluge</td>\n",
              "      <td>NaN</td>\n",
              "      <td>The f$&amp;amp;@ing things I do for #GISHWHES Just...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5448</th>\n",
              "      <td>7769</td>\n",
              "      <td>police</td>\n",
              "      <td>UK</td>\n",
              "      <td>DT @georgegalloway: RT @Galloway4Mayor: ÛÏThe...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>132</th>\n",
              "      <td>191</td>\n",
              "      <td>aftershock</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Aftershock back to school kick off was great. ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6845</th>\n",
              "      <td>9810</td>\n",
              "      <td>trauma</td>\n",
              "      <td>Montgomery County, MD</td>\n",
              "      <td>in response to trauma Children of Addicts deve...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ca387025-47ed-4ae4-abbd-13adc44b25ac')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-ca387025-47ed-4ae4-abbd-13adc44b25ac button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-ca387025-47ed-4ae4-abbd-13adc44b25ac');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-ae04f62b-f3d9-4d3f-8f24-9411656c9455\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ae04f62b-f3d9-4d3f-8f24-9411656c9455')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-ae04f62b-f3d9-4d3f-8f24-9411656c9455 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "train_df_shuffled",
              "summary": "{\n  \"name\": \"train_df_shuffled\",\n  \"rows\": 7613,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3137,\n        \"min\": 1,\n        \"max\": 10873,\n        \"num_unique_values\": 7613,\n        \"samples\": [\n          7061,\n          843,\n          10603\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"keyword\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 221,\n        \"samples\": [\n          \"blazing\",\n          \"emergency\",\n          \"rescue\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"location\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3341,\n        \"samples\": [\n          \"617-BTOWN-BEATDOWN\",\n          \"Newcastle Upon Tyne, England\",\n          \"Federal Capital Territory\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 7503,\n        \"samples\": [\n          \"This week first responders and DART members are participating in a four day intensive Technical Large Animal... http://t.co/tL93AOd3ER\",\n          \"My mom is watching a show about bridges breaking/falling and the people on them drowning in their cars aka one of my biggest fears ????\",\n          \"@NickLee8  i went to school in a bombed out East End of London3 families to one house no bathroom outside loo &amp; poor so whats yr point\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"target\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "huVFJH6ZkJpp",
        "outputId": "396f8cc6-5c31-4659-c97e-5442f28e47ba"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   id keyword location                                               text\n",
              "0   0     NaN      NaN                 Just happened a terrible car crash\n",
              "1   2     NaN      NaN  Heard about #earthquake is different cities, s...\n",
              "2   3     NaN      NaN  there is a forest fire at spot pond, geese are...\n",
              "3   9     NaN      NaN           Apocalypse lighting. #Spokane #wildfires\n",
              "4  11     NaN      NaN      Typhoon Soudelor kills 28 in China and Taiwan"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4253a77e-789b-4892-a941-1863aa18db97\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>keyword</th>\n",
              "      <th>location</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Just happened a terrible car crash</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Heard about #earthquake is different cities, s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>there is a forest fire at spot pond, geese are...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>9</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Apocalypse lighting. #Spokane #wildfires</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>11</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Typhoon Soudelor kills 28 in China and Taiwan</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4253a77e-789b-4892-a941-1863aa18db97')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-4253a77e-789b-4892-a941-1863aa18db97 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-4253a77e-789b-4892-a941-1863aa18db97');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-e2c08d6b-0aa2-4dd8-9207-979172cfe708\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-e2c08d6b-0aa2-4dd8-9207-979172cfe708')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-e2c08d6b-0aa2-4dd8-9207-979172cfe708 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "test_df",
              "summary": "{\n  \"name\": \"test_df\",\n  \"rows\": 3263,\n  \"fields\": [\n    {\n      \"column\": \"id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3146,\n        \"min\": 0,\n        \"max\": 10875,\n        \"num_unique_values\": 3263,\n        \"samples\": [\n          8051,\n          425,\n          1330\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"keyword\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 221,\n        \"samples\": [\n          \"injury\",\n          \"nuclear%20reactor\",\n          \"engulfed\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"location\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1602,\n        \"samples\": [\n          \"UAE\",\n          \"Tokio / Tokyo\",\n          \"Texas\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 3243,\n        \"samples\": [\n          \"Latest: USA: Huge sinkhole swallows up Brooklyn intersection http://t.co/vspKHg3nZy\",\n          \"I liked a @YouTube video http://t.co/a5YTAw9Vih S.O.S. Rona Guide - The Red Whirlwind\",\n          \"HitchBot travels Europe and greeted with open arms. Gets destroyed after two weeks in america. There's a lesson to be learned here.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_df.target.value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 178
        },
        "id": "Lcedp8rOkhiN",
        "outputId": "354b6f57-9972-409f-e712-3ca40458c8c5"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "target\n",
              "0    4342\n",
              "1    3271\n",
              "Name: count, dtype: int64"
            ],
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>count</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>target</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4342</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3271</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div><br><label><b>dtype:</b> int64</label>"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(f'Total training samples: {len(train_df)}')\n",
        "print(f'Total test samples: {len(test_df)}')\n",
        "print(f'Total samples: {len(train_df) + len(test_df)}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Zx1NpFHkpKu",
        "outputId": "544e6900-dda9-45fa-cf5b-6c470afd0526"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total training samples: 7613\n",
            "Total test samples: 3263\n",
            "Total samples: 10876\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "random_index = random.randint(0, len(train_df) - 5)\n",
        "for row in train_df_shuffled[['text', 'target']][random_index:random_index + 5].itertuples():\n",
        "  _, text, target = row\n",
        "  print(f'Target: {target}','(real disaster)' if target > 0 else '(not real disaster)')\n",
        "  print(f'Text:\\n{text}\\n')\n",
        "  print(f'---\\n')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "amCip3zik17Q",
        "outputId": "7c6cef08-7b85-4a17-f65d-f8171fe48efb"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Target: 0 (not real disaster)\n",
            "Text:\n",
            "i lava you! ????\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0 (not real disaster)\n",
            "Text:\n",
            "Men fear the feeling of being 'controlled' by a woman. Or passive aggressively being coerced into commitment before they're ready...\n",
            "\n",
            "---\n",
            "\n",
            "Target: 1 (real disaster)\n",
            "Text:\n",
            "@MichaelWestBiz standard damage control\n",
            "\n",
            "---\n",
            "\n",
            "Target: 1 (real disaster)\n",
            "Text:\n",
            "E1.1.2 Particulate=Break up of Solid Combust Fossil Fuel Voltaic Active Forest Fire Biological VOC=Petroleum CH4 Bacteria Decomposition\n",
            "\n",
            "---\n",
            "\n",
            "Target: 0 (not real disaster)\n",
            "Text:\n",
            "and I thought my surgical wounds were healed!!! this weather ain't helping either ):\n",
            "\n",
            "---\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "train_sentences, val_sentences, train_labels, val_labels = train_test_split(train_df_shuffled['text'].to_numpy(),\n",
        "                                                                            train_df_shuffled['target'].to_numpy(),\n",
        "                                                                            test_size = 0.1,\n",
        "                                                                            random_state = 42)"
      ],
      "metadata": {
        "id": "YNRbRCj_lo8E"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(train_sentences), len(train_labels), len(val_sentences), len(val_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x5fjNntwnGgY",
        "outputId": "fc1c08a0-839d-49ea-a377-e7cf667e399c"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(6851, 6851, 762, 762)"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_sentences[:10], train_labels[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BfeaVQ3cnOTF",
        "outputId": "8507beec-7faf-4205-bee0-1566f0e461da"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array(['@mogacola @zamtriossu i screamed after hitting tweet',\n",
              "        'Imagine getting flattened by Kurt Zouma',\n",
              "        '@Gurmeetramrahim #MSGDoing111WelfareWorks Green S welfare force ke appx 65000 members har time disaster victim ki help ke liye tyar hai....',\n",
              "        \"@shakjn @C7 @Magnums im shaking in fear he's gonna hack the planet\",\n",
              "        'Somehow find you and I collide http://t.co/Ee8RpOahPk',\n",
              "        '@EvaHanderek @MarleyKnysh great times until the bus driver held us hostage in the mall parking lot lmfao',\n",
              "        'destroy the free fandom honestly',\n",
              "        'Weapons stolen from National Guard Armory in New Albany still missing #Gunsense http://t.co/lKNU8902JE',\n",
              "        '@wfaaweather Pete when will the heat wave pass? Is it really going to be mid month? Frisco Boy Scouts have a canoe trip in Okla.',\n",
              "        'Patient-reported outcomes in long-term survivors of metastatic colorectal cancer - British Journal of Surgery http://t.co/5Yl4DC1Tqt'],\n",
              "       dtype=object),\n",
              " array([0, 0, 1, 0, 0, 1, 1, 0, 1, 1]))"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras.layers import TextVectorization\n",
        "\n",
        "text_vectorizer = TextVectorization(max_tokens = None,\n",
        "                                    standardize = 'lower_and_strip_punctuation',\n",
        "                                    split = 'whitespace',\n",
        "                                    ngrams = None,\n",
        "                                    output_mode = 'int',\n",
        "                                    output_sequence_length = None)"
      ],
      "metadata": {
        "id": "R7ebwEBanUtK"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "round(sum([len(i.split()) for i in train_sentences]) / len(train_sentences))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GpUUAK-Etsso",
        "outputId": "ac4f2dab-5aa3-4d3e-9745-f8ae83d75478"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "15"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "max_vocab_length = 10000\n",
        "max_length = 15\n",
        "text_vectorizer = TextVectorization(max_tokens = max_vocab_length,\n",
        "                                    output_mode = 'int',\n",
        "                                    output_sequence_length = max_length)"
      ],
      "metadata": {
        "id": "xbuQDHUZuR8Y"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text_vectorizer.adapt(train_sentences)"
      ],
      "metadata": {
        "id": "IqWH8SNPuffe"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sample_sentence = \"There's a flood in my street!\"\n",
        "text_vectorizer([sample_sentence])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oQw7Gw7UunPU",
        "outputId": "4d047a7c-3075-45f7-da6b-0d97b2bea9d4"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 15), dtype=int64, numpy=\n",
              "array([[264,   3, 232,   4,  13, 698,   0,   0,   0,   0,   0,   0,   0,\n",
              "          0,   0]])>"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "random_sentence = random.choice(train_sentences)\n",
        "print(f\"Original Sentence1:\\n{random_sentence}\\n\\nVectorized version: \")\n",
        "text_vectorizer([random_sentence])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jlnBHSpzuyJo",
        "outputId": "59267ec0-74ed-4345-974a-17e04313f6be"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original Sentence1:\n",
            "Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/H84R1TIh8J\n",
            "\n",
            "Vectorized version: \n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 15), dtype=int64, numpy=\n",
              "array([[135,  19, 559,  52, 640,  27,  18,   3, 365,   4, 204,   3, 539,\n",
              "          1,   0]])>"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "words_in_vocab = text_vectorizer.get_vocabulary()\n",
        "top_5_words = words_in_vocab[:5]\n",
        "bottom_5_words = words_in_vocab[-5:]\n",
        "print(f'Number of words in vocab: {len(words_in_vocab)}')\n",
        "print(f'Top 5 most common words : {top_5_words}')\n",
        "print(f'Bottom 5 least common words : {bottom_5_words}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "azqW29y0vn7B",
        "outputId": "ad031f1e-f155-4f01-f08b-e89c23079050"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of words in vocab: 10000\n",
            "Top 5 most common words : ['', '[UNK]', 'the', 'a', 'in']\n",
            "Bottom 5 least common words : ['pages', 'paeds', 'pads', 'padres', 'paddytomlinson1']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tf.random.set_seed(42)\n",
        "from tensorflow.keras import layers\n",
        "\n",
        "embedding = layers.Embedding(input_dim = max_vocab_length,\n",
        "                             output_dim = 128,\n",
        "                             embeddings_initializer = 'uniform',\n",
        "                             input_length = max_length,\n",
        "                             name = 'embedding_1')\n",
        "\n",
        "embedding"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nSfEPIaewbpT",
        "outputId": "43e12c1f-1b6b-4ed6-a31c-a5ae0657765c"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/keras/src/layers/core/embedding.py:90: UserWarning: Argument `input_length` is deprecated. Just remove it.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Embedding name=embedding_1, built=False>"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "random_sentence = random.choice(train_sentences)\n",
        "print(f\"Original Sentence: \\n{random_sentence}\\n Embedding:\")\n",
        "sample_embed = embedding(text_vectorizer([random_sentence]))\n",
        "sample_embed"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bKPKnvwpxXSE",
        "outputId": "86a9f39a-f664-4b1a-ccf2-d2eee56a11cc"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original Sentence: \n",
            "Still blazing ????\n",
            " Embedding:\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 15, 128), dtype=float32, numpy=\n",
              "array([[[ 0.02263899,  0.04282571, -0.00747859, ..., -0.01059432,\n",
              "         -0.0230093 , -0.00547081],\n",
              "        [-0.01697371,  0.04455257,  0.02342946, ..., -0.00498071,\n",
              "         -0.02057469, -0.02670168],\n",
              "        [-0.01314722,  0.02784585,  0.04983481, ...,  0.02798307,\n",
              "          0.00844678,  0.01084675],\n",
              "        ...,\n",
              "        [-0.01314722,  0.02784585,  0.04983481, ...,  0.02798307,\n",
              "          0.00844678,  0.01084675],\n",
              "        [-0.01314722,  0.02784585,  0.04983481, ...,  0.02798307,\n",
              "          0.00844678,  0.01084675],\n",
              "        [-0.01314722,  0.02784585,  0.04983481, ...,  0.02798307,\n",
              "          0.00844678,  0.01084675]]], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sample_embed[0][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R3bo__3qxslT",
        "outputId": "b086fd2e-0e4c-4ecd-fbce-d01c68139e21"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
              "array([ 0.02263899,  0.04282571, -0.00747859,  0.00089192,  0.04500872,\n",
              "       -0.00142722,  0.04726816,  0.03628168,  0.04347006, -0.04645012,\n",
              "        0.02594068,  0.00431327, -0.04583221, -0.01125815,  0.00808363,\n",
              "        0.00415226,  0.04027236, -0.04296472, -0.02459514,  0.02741987,\n",
              "        0.00120406, -0.03918559,  0.00112037,  0.03599444,  0.01047128,\n",
              "        0.00643545, -0.02384254, -0.01367823, -0.02450765, -0.01069454,\n",
              "       -0.0027113 , -0.04961702, -0.03108518, -0.04341054, -0.01487225,\n",
              "       -0.00901305, -0.02664474, -0.01494026,  0.02640976,  0.02982089,\n",
              "       -0.01717597,  0.00558516,  0.00045701, -0.02245234, -0.00450317,\n",
              "       -0.00513272, -0.04660535,  0.01655433,  0.00588652,  0.04959622,\n",
              "       -0.00338999, -0.02922297, -0.00603061,  0.02842215, -0.03484003,\n",
              "        0.0075462 , -0.02218043, -0.0495839 , -0.00985516, -0.0460765 ,\n",
              "        0.0102103 , -0.0208317 , -0.02729491,  0.04335861,  0.02940992,\n",
              "        0.00046239,  0.03651584,  0.00645981, -0.02389184,  0.01586041,\n",
              "       -0.04815722, -0.00674069, -0.00388305, -0.03043283,  0.01556803,\n",
              "       -0.02960004, -0.02625467, -0.00935595, -0.03676865, -0.01827329,\n",
              "       -0.03114669,  0.03085576,  0.03754828,  0.00824984,  0.04767153,\n",
              "       -0.04939734, -0.01018097,  0.00704141,  0.04093562, -0.04819141,\n",
              "        0.01150514,  0.01121583, -0.02820104,  0.04787037, -0.03436478,\n",
              "       -0.03298855, -0.0135316 ,  0.02911011,  0.03608713,  0.0458777 ,\n",
              "       -0.0342217 ,  0.03313539, -0.00499848, -0.0119753 ,  0.01008735,\n",
              "        0.03100314, -0.01533407,  0.01664387,  0.02917613, -0.03982633,\n",
              "       -0.03286426, -0.04486734, -0.01781883, -0.00228886, -0.02349469,\n",
              "        0.02717629,  0.0425575 ,  0.02389748,  0.0356745 , -0.00164407,\n",
              "       -0.0395235 ,  0.00476211, -0.02583328,  0.04280623,  0.03731092,\n",
              "       -0.01059432, -0.0230093 , -0.00547081], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "model_0 = Pipeline([\n",
        "    ('tfidf', TfidfVectorizer()),\n",
        "    ('clf', MultinomialNB())\n",
        "])\n",
        "\n",
        "model_0.fit(train_sentences, train_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        },
        "id": "xAGhnFmXyAep",
        "outputId": "9c72fdd2-4926-427b-f328-bb7b79bd1721"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('tfidf', TfidfVectorizer()), ('clf', MultinomialNB())])"
            ],
            "text/html": [
              "<style>#sk-container-id-1 {\n",
              "  /* Definition of color scheme common for light and dark mode */\n",
              "  --sklearn-color-text: black;\n",
              "  --sklearn-color-line: gray;\n",
              "  /* Definition of color scheme for unfitted estimators */\n",
              "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
              "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
              "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
              "  --sklearn-color-unfitted-level-3: chocolate;\n",
              "  /* Definition of color scheme for fitted estimators */\n",
              "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
              "  --sklearn-color-fitted-level-1: #d4ebff;\n",
              "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
              "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
              "\n",
              "  /* Specific color for light theme */\n",
              "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
              "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
              "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
              "  --sklearn-color-icon: #696969;\n",
              "\n",
              "  @media (prefers-color-scheme: dark) {\n",
              "    /* Redefinition of color scheme for dark theme */\n",
              "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
              "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
              "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
              "    --sklearn-color-icon: #878787;\n",
              "  }\n",
              "}\n",
              "\n",
              "#sk-container-id-1 {\n",
              "  color: var(--sklearn-color-text);\n",
              "}\n",
              "\n",
              "#sk-container-id-1 pre {\n",
              "  padding: 0;\n",
              "}\n",
              "\n",
              "#sk-container-id-1 input.sk-hidden--visually {\n",
              "  border: 0;\n",
              "  clip: rect(1px 1px 1px 1px);\n",
              "  clip: rect(1px, 1px, 1px, 1px);\n",
              "  height: 1px;\n",
              "  margin: -1px;\n",
              "  overflow: hidden;\n",
              "  padding: 0;\n",
              "  position: absolute;\n",
              "  width: 1px;\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-dashed-wrapped {\n",
              "  border: 1px dashed var(--sklearn-color-line);\n",
              "  margin: 0 0.4em 0.5em 0.4em;\n",
              "  box-sizing: border-box;\n",
              "  padding-bottom: 0.4em;\n",
              "  background-color: var(--sklearn-color-background);\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-container {\n",
              "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
              "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
              "     so we also need the `!important` here to be able to override the\n",
              "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
              "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
              "  display: inline-block !important;\n",
              "  position: relative;\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-text-repr-fallback {\n",
              "  display: none;\n",
              "}\n",
              "\n",
              "div.sk-parallel-item,\n",
              "div.sk-serial,\n",
              "div.sk-item {\n",
              "  /* draw centered vertical line to link estimators */\n",
              "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
              "  background-size: 2px 100%;\n",
              "  background-repeat: no-repeat;\n",
              "  background-position: center center;\n",
              "}\n",
              "\n",
              "/* Parallel-specific style estimator block */\n",
              "\n",
              "#sk-container-id-1 div.sk-parallel-item::after {\n",
              "  content: \"\";\n",
              "  width: 100%;\n",
              "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
              "  flex-grow: 1;\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-parallel {\n",
              "  display: flex;\n",
              "  align-items: stretch;\n",
              "  justify-content: center;\n",
              "  background-color: var(--sklearn-color-background);\n",
              "  position: relative;\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-parallel-item {\n",
              "  display: flex;\n",
              "  flex-direction: column;\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-parallel-item:first-child::after {\n",
              "  align-self: flex-end;\n",
              "  width: 50%;\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-parallel-item:last-child::after {\n",
              "  align-self: flex-start;\n",
              "  width: 50%;\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-parallel-item:only-child::after {\n",
              "  width: 0;\n",
              "}\n",
              "\n",
              "/* Serial-specific style estimator block */\n",
              "\n",
              "#sk-container-id-1 div.sk-serial {\n",
              "  display: flex;\n",
              "  flex-direction: column;\n",
              "  align-items: center;\n",
              "  background-color: var(--sklearn-color-background);\n",
              "  padding-right: 1em;\n",
              "  padding-left: 1em;\n",
              "}\n",
              "\n",
              "\n",
              "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
              "clickable and can be expanded/collapsed.\n",
              "- Pipeline and ColumnTransformer use this feature and define the default style\n",
              "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
              "*/\n",
              "\n",
              "/* Pipeline and ColumnTransformer style (default) */\n",
              "\n",
              "#sk-container-id-1 div.sk-toggleable {\n",
              "  /* Default theme specific background. It is overwritten whether we have a\n",
              "  specific estimator or a Pipeline/ColumnTransformer */\n",
              "  background-color: var(--sklearn-color-background);\n",
              "}\n",
              "\n",
              "/* Toggleable label */\n",
              "#sk-container-id-1 label.sk-toggleable__label {\n",
              "  cursor: pointer;\n",
              "  display: block;\n",
              "  width: 100%;\n",
              "  margin-bottom: 0;\n",
              "  padding: 0.5em;\n",
              "  box-sizing: border-box;\n",
              "  text-align: center;\n",
              "}\n",
              "\n",
              "#sk-container-id-1 label.sk-toggleable__label-arrow:before {\n",
              "  /* Arrow on the left of the label */\n",
              "  content: \"▸\";\n",
              "  float: left;\n",
              "  margin-right: 0.25em;\n",
              "  color: var(--sklearn-color-icon);\n",
              "}\n",
              "\n",
              "#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {\n",
              "  color: var(--sklearn-color-text);\n",
              "}\n",
              "\n",
              "/* Toggleable content - dropdown */\n",
              "\n",
              "#sk-container-id-1 div.sk-toggleable__content {\n",
              "  max-height: 0;\n",
              "  max-width: 0;\n",
              "  overflow: hidden;\n",
              "  text-align: left;\n",
              "  /* unfitted */\n",
              "  background-color: var(--sklearn-color-unfitted-level-0);\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-toggleable__content.fitted {\n",
              "  /* fitted */\n",
              "  background-color: var(--sklearn-color-fitted-level-0);\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-toggleable__content pre {\n",
              "  margin: 0.2em;\n",
              "  border-radius: 0.25em;\n",
              "  color: var(--sklearn-color-text);\n",
              "  /* unfitted */\n",
              "  background-color: var(--sklearn-color-unfitted-level-0);\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-toggleable__content.fitted pre {\n",
              "  /* unfitted */\n",
              "  background-color: var(--sklearn-color-fitted-level-0);\n",
              "}\n",
              "\n",
              "#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
              "  /* Expand drop-down */\n",
              "  max-height: 200px;\n",
              "  max-width: 100%;\n",
              "  overflow: auto;\n",
              "}\n",
              "\n",
              "#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
              "  content: \"▾\";\n",
              "}\n",
              "\n",
              "/* Pipeline/ColumnTransformer-specific style */\n",
              "\n",
              "#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
              "  color: var(--sklearn-color-text);\n",
              "  background-color: var(--sklearn-color-unfitted-level-2);\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
              "  background-color: var(--sklearn-color-fitted-level-2);\n",
              "}\n",
              "\n",
              "/* Estimator-specific style */\n",
              "\n",
              "/* Colorize estimator box */\n",
              "#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
              "  /* unfitted */\n",
              "  background-color: var(--sklearn-color-unfitted-level-2);\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
              "  /* fitted */\n",
              "  background-color: var(--sklearn-color-fitted-level-2);\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-label label.sk-toggleable__label,\n",
              "#sk-container-id-1 div.sk-label label {\n",
              "  /* The background is the default theme color */\n",
              "  color: var(--sklearn-color-text-on-default-background);\n",
              "}\n",
              "\n",
              "/* On hover, darken the color of the background */\n",
              "#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {\n",
              "  color: var(--sklearn-color-text);\n",
              "  background-color: var(--sklearn-color-unfitted-level-2);\n",
              "}\n",
              "\n",
              "/* Label box, darken color on hover, fitted */\n",
              "#sk-container-id-1 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
              "  color: var(--sklearn-color-text);\n",
              "  background-color: var(--sklearn-color-fitted-level-2);\n",
              "}\n",
              "\n",
              "/* Estimator label */\n",
              "\n",
              "#sk-container-id-1 div.sk-label label {\n",
              "  font-family: monospace;\n",
              "  font-weight: bold;\n",
              "  display: inline-block;\n",
              "  line-height: 1.2em;\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-label-container {\n",
              "  text-align: center;\n",
              "}\n",
              "\n",
              "/* Estimator-specific */\n",
              "#sk-container-id-1 div.sk-estimator {\n",
              "  font-family: monospace;\n",
              "  border: 1px dotted var(--sklearn-color-border-box);\n",
              "  border-radius: 0.25em;\n",
              "  box-sizing: border-box;\n",
              "  margin-bottom: 0.5em;\n",
              "  /* unfitted */\n",
              "  background-color: var(--sklearn-color-unfitted-level-0);\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-estimator.fitted {\n",
              "  /* fitted */\n",
              "  background-color: var(--sklearn-color-fitted-level-0);\n",
              "}\n",
              "\n",
              "/* on hover */\n",
              "#sk-container-id-1 div.sk-estimator:hover {\n",
              "  /* unfitted */\n",
              "  background-color: var(--sklearn-color-unfitted-level-2);\n",
              "}\n",
              "\n",
              "#sk-container-id-1 div.sk-estimator.fitted:hover {\n",
              "  /* fitted */\n",
              "  background-color: var(--sklearn-color-fitted-level-2);\n",
              "}\n",
              "\n",
              "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
              "\n",
              "/* Common style for \"i\" and \"?\" */\n",
              "\n",
              ".sk-estimator-doc-link,\n",
              "a:link.sk-estimator-doc-link,\n",
              "a:visited.sk-estimator-doc-link {\n",
              "  float: right;\n",
              "  font-size: smaller;\n",
              "  line-height: 1em;\n",
              "  font-family: monospace;\n",
              "  background-color: var(--sklearn-color-background);\n",
              "  border-radius: 1em;\n",
              "  height: 1em;\n",
              "  width: 1em;\n",
              "  text-decoration: none !important;\n",
              "  margin-left: 1ex;\n",
              "  /* unfitted */\n",
              "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
              "  color: var(--sklearn-color-unfitted-level-1);\n",
              "}\n",
              "\n",
              ".sk-estimator-doc-link.fitted,\n",
              "a:link.sk-estimator-doc-link.fitted,\n",
              "a:visited.sk-estimator-doc-link.fitted {\n",
              "  /* fitted */\n",
              "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
              "  color: var(--sklearn-color-fitted-level-1);\n",
              "}\n",
              "\n",
              "/* On hover */\n",
              "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
              ".sk-estimator-doc-link:hover,\n",
              "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
              ".sk-estimator-doc-link:hover {\n",
              "  /* unfitted */\n",
              "  background-color: var(--sklearn-color-unfitted-level-3);\n",
              "  color: var(--sklearn-color-background);\n",
              "  text-decoration: none;\n",
              "}\n",
              "\n",
              "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
              ".sk-estimator-doc-link.fitted:hover,\n",
              "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
              ".sk-estimator-doc-link.fitted:hover {\n",
              "  /* fitted */\n",
              "  background-color: var(--sklearn-color-fitted-level-3);\n",
              "  color: var(--sklearn-color-background);\n",
              "  text-decoration: none;\n",
              "}\n",
              "\n",
              "/* Span, style for the box shown on hovering the info icon */\n",
              ".sk-estimator-doc-link span {\n",
              "  display: none;\n",
              "  z-index: 9999;\n",
              "  position: relative;\n",
              "  font-weight: normal;\n",
              "  right: .2ex;\n",
              "  padding: .5ex;\n",
              "  margin: .5ex;\n",
              "  width: min-content;\n",
              "  min-width: 20ex;\n",
              "  max-width: 50ex;\n",
              "  color: var(--sklearn-color-text);\n",
              "  box-shadow: 2pt 2pt 4pt #999;\n",
              "  /* unfitted */\n",
              "  background: var(--sklearn-color-unfitted-level-0);\n",
              "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
              "}\n",
              "\n",
              ".sk-estimator-doc-link.fitted span {\n",
              "  /* fitted */\n",
              "  background: var(--sklearn-color-fitted-level-0);\n",
              "  border: var(--sklearn-color-fitted-level-3);\n",
              "}\n",
              "\n",
              ".sk-estimator-doc-link:hover span {\n",
              "  display: block;\n",
              "}\n",
              "\n",
              "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
              "\n",
              "#sk-container-id-1 a.estimator_doc_link {\n",
              "  float: right;\n",
              "  font-size: 1rem;\n",
              "  line-height: 1em;\n",
              "  font-family: monospace;\n",
              "  background-color: var(--sklearn-color-background);\n",
              "  border-radius: 1rem;\n",
              "  height: 1rem;\n",
              "  width: 1rem;\n",
              "  text-decoration: none;\n",
              "  /* unfitted */\n",
              "  color: var(--sklearn-color-unfitted-level-1);\n",
              "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
              "}\n",
              "\n",
              "#sk-container-id-1 a.estimator_doc_link.fitted {\n",
              "  /* fitted */\n",
              "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
              "  color: var(--sklearn-color-fitted-level-1);\n",
              "}\n",
              "\n",
              "/* On hover */\n",
              "#sk-container-id-1 a.estimator_doc_link:hover {\n",
              "  /* unfitted */\n",
              "  background-color: var(--sklearn-color-unfitted-level-3);\n",
              "  color: var(--sklearn-color-background);\n",
              "  text-decoration: none;\n",
              "}\n",
              "\n",
              "#sk-container-id-1 a.estimator_doc_link.fitted:hover {\n",
              "  /* fitted */\n",
              "  background-color: var(--sklearn-color-fitted-level-3);\n",
              "}\n",
              "</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;tfidf&#x27;, TfidfVectorizer()), (&#x27;clf&#x27;, MultinomialNB())])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;&nbsp;Pipeline<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.pipeline.Pipeline.html\">?<span>Documentation for Pipeline</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>Pipeline(steps=[(&#x27;tfidf&#x27;, TfidfVectorizer()), (&#x27;clf&#x27;, MultinomialNB())])</pre></div> </div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;TfidfVectorizer<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html\">?<span>Documentation for TfidfVectorizer</span></a></label><div class=\"sk-toggleable__content fitted\"><pre>TfidfVectorizer()</pre></div> </div></div><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;MultinomialNB<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.naive_bayes.MultinomialNB.html\">?<span>Documentation for MultinomialNB</span></a></label><div class=\"sk-toggleable__content fitted\"><pre>MultinomialNB()</pre></div> </div></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "baseline_score = model_0.score(val_sentences, val_labels)\n",
        "print(f'Our baseline model achieves an accuracy of: {baseline_score*100:.2f}%')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h7Gno0yDzSBj",
        "outputId": "c49f7ddc-76a0-4527-e559-5d4f102bd8e5"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Our baseline model achieves an accuracy of: 79.27%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "baseline_preds = model_0.predict(val_sentences)\n",
        "baseline_preds"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZsIacO61zheN",
        "outputId": "de608ca8-bc67-4188-e2a1-f7e5c6e72b83"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0,\n",
              "       0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0,\n",
              "       0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 1,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0,\n",
              "       1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0,\n",
              "       1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1,\n",
              "       1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0,\n",
              "       0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,\n",
              "       0, 0, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,\n",
              "       1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0,\n",
              "       1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0,\n",
              "       1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1,\n",
              "       1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0,\n",
              "       1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,\n",
              "       0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1,\n",
              "       0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1,\n",
              "       0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1,\n",
              "       0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1,\n",
              "       0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0,\n",
              "       0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1,\n",
              "       0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0,\n",
              "       0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0,\n",
              "       0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0,\n",
              "       0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1,\n",
              "       1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0,\n",
              "       0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0,\n",
              "       0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0,\n",
              "       0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0,\n",
              "       0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0,\n",
              "       0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0])"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
        "\n",
        "def calculate_results(y_true, y_pred):\n",
        "  model_accuracy = accuracy_score(y_true, y_pred) * 100\n",
        "  model_precision, model_recall, model_f1, _ = precision_recall_fscore_support(y_true, y_pred, average = 'weighted')\n",
        "  model_results = {'accuracy' : model_accuracy,\n",
        "                   'precision' : model_precision,\n",
        "                   'recall' : model_recall,\n",
        "                   'f1' : model_f1}\n",
        "  return model_results"
      ],
      "metadata": {
        "id": "UWuIPR_VznHl"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "baseline_results = calculate_results(val_labels, baseline_preds)\n",
        "baseline_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DzPEpeGy0Zde",
        "outputId": "e7356fb1-be24-47ed-8e62-a49b4850064b"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 79.26509186351706,\n",
              " 'precision': 0.8111390004213173,\n",
              " 'recall': 0.7926509186351706,\n",
              " 'f1': 0.7862189758049549}"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "SAVE_DIR = 'model_logs'"
      ],
      "metadata": {
        "id": "787SBiqD0heQ"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "inputs = layers.Input(shape = (1,), dtype = 'string')\n",
        "x = text_vectorizer(inputs)\n",
        "x = embedding(x)\n",
        "x = layers.GlobalAveragePooling1D()(x)\n",
        "outputs = layers.Dense(1, activation = 'sigmoid')(x)\n",
        "model_1 = tf.keras.Model(inputs, outputs, name = 'model_1_dense')"
      ],
      "metadata": {
        "id": "iSJiM8BU03YS"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_1.compile(loss = 'binary_crossentropy',\n",
        "                optimizer = 'adam',\n",
        "                metrics = ['accuracy'])"
      ],
      "metadata": {
        "id": "Rsrr96ES1Rd_"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_1.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 321
        },
        "id": "WEPA3nB814Tm",
        "outputId": "861855b5-c979-42bd-92bc-18353124efda"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1mModel: \"model_1_dense\"\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"model_1_dense\"</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
              "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                        \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape               \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m        Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
              "│ input_layer (\u001b[38;5;33mInputLayer\u001b[0m)             │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)                   │               \u001b[38;5;34m0\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ text_vectorization_1                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m15\u001b[0m)                  │               \u001b[38;5;34m0\u001b[0m │\n",
              "│ (\u001b[38;5;33mTextVectorization\u001b[0m)                  │                             │                 │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ embedding_1 (\u001b[38;5;33mEmbedding\u001b[0m)              │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m15\u001b[0m, \u001b[38;5;34m128\u001b[0m)             │       \u001b[38;5;34m1,280,000\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ global_average_pooling1d             │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)                 │               \u001b[38;5;34m0\u001b[0m │\n",
              "│ (\u001b[38;5;33mGlobalAveragePooling1D\u001b[0m)             │                             │                 │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ dense (\u001b[38;5;33mDense\u001b[0m)                        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)                   │             \u001b[38;5;34m129\u001b[0m │\n",
              "└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
              "┃<span style=\"font-weight: bold\"> Layer (type)                         </span>┃<span style=\"font-weight: bold\"> Output Shape                </span>┃<span style=\"font-weight: bold\">         Param # </span>┃\n",
              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
              "│ input_layer (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)             │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)                   │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ text_vectorization_1                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">15</span>)                  │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TextVectorization</span>)                  │                             │                 │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ embedding_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)              │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">15</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)             │       <span style=\"color: #00af00; text-decoration-color: #00af00\">1,280,000</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ global_average_pooling1d             │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)                 │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GlobalAveragePooling1D</span>)             │                             │                 │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ dense (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)                   │             <span style=\"color: #00af00; text-decoration-color: #00af00\">129</span> │\n",
              "└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m1,280,129\u001b[0m (4.88 MB)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">1,280,129</span> (4.88 MB)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m1,280,129\u001b[0m (4.88 MB)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">1,280,129</span> (4.88 MB)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_1_history = model_1.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs = 5,\n",
        "                              validation_data = (val_sentences, val_labels),\n",
        "                              callbacks = [create_tensorboard_callback(dir_name = SAVE_DIR,\n",
        "                                                                       experiment_name = 'simple_dense_model')])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IcSsxUSh15jm",
        "outputId": "8cfd00b8-3a71-4697-d1e8-1fb86e0b1a87"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/simple_dense_model/20241218-140052\n",
            "Epoch 1/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m9s\u001b[0m 19ms/step - accuracy: 0.6330 - loss: 0.6513 - val_accuracy: 0.7598 - val_loss: 0.5343\n",
            "Epoch 2/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 16ms/step - accuracy: 0.8074 - loss: 0.4680 - val_accuracy: 0.7900 - val_loss: 0.4731\n",
            "Epoch 3/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m6s\u001b[0m 19ms/step - accuracy: 0.8534 - loss: 0.3634 - val_accuracy: 0.7979 - val_loss: 0.4607\n",
            "Epoch 4/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 18ms/step - accuracy: 0.8866 - loss: 0.2969 - val_accuracy: 0.7900 - val_loss: 0.4666\n",
            "Epoch 5/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 15ms/step - accuracy: 0.9054 - loss: 0.2480 - val_accuracy: 0.7808 - val_loss: 0.4821\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_1.evaluate(val_sentences, val_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yA9qBP4O25sp",
        "outputId": "f6dce419-a54f-4816-fd06-2958004d223f"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m24/24\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7690 - loss: 0.5140 \n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.4820725619792938, 0.7808399200439453]"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embedding.weights"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WfvnKL5t3Bqz",
        "outputId": "f6df5c03-2fe6-4692-f35d-f73443d13154"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<KerasVariable shape=(10000, 128), dtype=float32, path=embedding_1/embeddings>]"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embed_weights = model_1.get_layer('embedding_1').get_weights()[0]\n",
        "print(embed_weights.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1h6TTRDs3Fs6",
        "outputId": "acba2278-08cd-405d-f410-e75b62aaba00"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 128)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!tensorboard dev upload --logdir ./model_logs \\\n",
        "  --name \"First deep model on text data\" \\\n",
        "  --description \"Trying a dense model with an embedding layer\" \\\n",
        "  --one_shot"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gvVJC0bp3UsI",
        "outputId": "ec978d17-3a96-4f15-ad97-7dee8d5ccb56"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-12-18 14:01:19.442296: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-12-18 14:01:19.467593: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-12-18 14:01:19.474956: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-12-18 14:01:20.895270: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "======================================================================\n",
            "ERROR: The `tensorboard dev` command is no longer available.\n",
            "\n",
            "TensorBoard.dev has been shut down. For further information,\n",
            "see the FAQ at <https://tensorboard.dev/>.\n",
            "======================================================================\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_1_pred_probs = model_1.predict(val_sentences)\n",
        "model_1_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0My70k2h3sDE",
        "outputId": "f8a3eb31-9dc0-4393-89f7-86e833e0ff78"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m24/24\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 11ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.33567098],\n",
              "       [0.734848  ],\n",
              "       [0.9978247 ],\n",
              "       [0.1920529 ],\n",
              "       [0.1004472 ],\n",
              "       [0.94609135],\n",
              "       [0.892815  ],\n",
              "       [0.9943365 ],\n",
              "       [0.9609088 ],\n",
              "       [0.3790033 ]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_1_preds = tf.squeeze(tf.round(model_1_pred_probs))\n",
        "model_1_preds[:20]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BMQEDsdi3-n0",
        "outputId": "2f90d311-f71e-4d67-e0bf-fe1a861429a8"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(20,), dtype=float32, numpy=\n",
              "array([0., 1., 1., 0., 0., 1., 1., 1., 1., 0., 0., 1., 0., 0., 0., 0., 0.,\n",
              "       0., 0., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_1_results = calculate_results(val_labels, model_1_preds)\n",
        "model_1_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vbl8GmAq4S-W",
        "outputId": "d5b6b56b-ad27-4940-ceb8-8ab2f134714c"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 78.08398950131233,\n",
              " 'precision': 0.7841274438015813,\n",
              " 'recall': 0.7808398950131233,\n",
              " 'f1': 0.7782630349987989}"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "np.array(list(model_1_results.values())) > np.array(list(baseline_results.values()))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JHkve_5u4dzk",
        "outputId": "8c935dee-3575-463a-cfdc-ed9b217cdbe0"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([False, False, False, False])"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def compare_baseline_to_new_results(baseline_results, new_model_results):\n",
        "  for key, value in baseline_results.items():\n",
        "    print(f'Baseline {key}:{value:.2f}, New {key}: {new_model_results[key]:.2f}, Difference: {new_model_results[key] - value:.2f}')\n",
        "\n",
        "compare_baseline_to_new_results(baseline_results = baseline_results,\n",
        "                                new_model_results = model_1_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S9M5qmeY4sht",
        "outputId": "ec057eaf-983b-47d0-9e0f-afeb9ad7f5a1"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Baseline accuracy:79.27, New accuracy: 78.08, Difference: -1.18\n",
            "Baseline precision:0.81, New precision: 0.78, Difference: -0.03\n",
            "Baseline recall:0.79, New recall: 0.78, Difference: -0.01\n",
            "Baseline f1:0.79, New f1: 0.78, Difference: -0.01\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tf.random.set_seed(42)\n",
        "\n",
        "model_2_embedding = layers.Embedding(input_dim = max_vocab_length,\n",
        "                                     output_dim = 128,\n",
        "                                     embeddings_initializer = 'uniform',\n",
        "                                     input_length = max_length,\n",
        "                                     name = 'embedding_2')\n",
        "\n",
        "inputs = layers.Input(shape = (1, ), dtype = 'string')\n",
        "x = text_vectorizer(inputs)\n",
        "x = model_2_embedding(x)\n",
        "print(x.shape)\n",
        "x = layers.LSTM(64)(x)\n",
        "outputs = layers.Dense(1, activation = 'sigmoid')(x)\n",
        "model_2 = tf.keras.Model(inputs, outputs, name = 'model_2_LSTM')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J0cmxijI5ar-",
        "outputId": "ed174252-b081-469a-c7a7-354d1974ef7a"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(None, 15, 128)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/keras/src/layers/core/embedding.py:90: UserWarning: Argument `input_length` is deprecated. Just remove it.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_2.compile(loss = 'binary_crossentropy',\n",
        "                optimizer = 'adam',\n",
        "                metrics = ['accuracy'])"
      ],
      "metadata": {
        "id": "gercIRhcKBkD"
      },
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_2.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 305
        },
        "id": "wLbC6ywmK26q",
        "outputId": "1d7588f2-7164-40cd-dd42-6dd258d713cd"
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1mModel: \"model_2_LSTM\"\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"model_2_LSTM\"</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
              "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                        \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape               \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m        Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
              "│ input_layer_1 (\u001b[38;5;33mInputLayer\u001b[0m)           │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)                   │               \u001b[38;5;34m0\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ text_vectorization_1                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m15\u001b[0m)                  │               \u001b[38;5;34m0\u001b[0m │\n",
              "│ (\u001b[38;5;33mTextVectorization\u001b[0m)                  │                             │                 │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ embedding_2 (\u001b[38;5;33mEmbedding\u001b[0m)              │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m15\u001b[0m, \u001b[38;5;34m128\u001b[0m)             │       \u001b[38;5;34m1,280,000\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ lstm (\u001b[38;5;33mLSTM\u001b[0m)                          │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m64\u001b[0m)                  │          \u001b[38;5;34m49,408\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ dense_1 (\u001b[38;5;33mDense\u001b[0m)                      │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)                   │              \u001b[38;5;34m65\u001b[0m │\n",
              "└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
              "┃<span style=\"font-weight: bold\"> Layer (type)                         </span>┃<span style=\"font-weight: bold\"> Output Shape                </span>┃<span style=\"font-weight: bold\">         Param # </span>┃\n",
              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
              "│ input_layer_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)           │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)                   │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ text_vectorization_1                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">15</span>)                  │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TextVectorization</span>)                  │                             │                 │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ embedding_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)              │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">15</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)             │       <span style=\"color: #00af00; text-decoration-color: #00af00\">1,280,000</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ lstm (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">LSTM</span>)                          │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)                  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">49,408</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ dense_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                      │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)                   │              <span style=\"color: #00af00; text-decoration-color: #00af00\">65</span> │\n",
              "└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m1,329,473\u001b[0m (5.07 MB)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">1,329,473</span> (5.07 MB)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m1,329,473\u001b[0m (5.07 MB)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">1,329,473</span> (5.07 MB)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_2_history = model_2.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs = 5,\n",
        "                              validation_data = (val_sentences, val_labels),\n",
        "                              callbacks = [create_tensorboard_callback(SAVE_DIR, 'LSTM')])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fLwcIj6NK823",
        "outputId": "b59f57e6-92e0-4470-cc48-4eb3d89ec0e4"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/LSTM/20241218-140124\n",
            "Epoch 1/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m9s\u001b[0m 25ms/step - accuracy: 0.6615 - loss: 0.5826 - val_accuracy: 0.7730 - val_loss: 0.4598\n",
            "Epoch 2/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 24ms/step - accuracy: 0.8654 - loss: 0.3312 - val_accuracy: 0.7572 - val_loss: 0.5034\n",
            "Epoch 3/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 25ms/step - accuracy: 0.9179 - loss: 0.2268 - val_accuracy: 0.7638 - val_loss: 0.6111\n",
            "Epoch 4/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 24ms/step - accuracy: 0.9499 - loss: 0.1512 - val_accuracy: 0.7572 - val_loss: 0.6701\n",
            "Epoch 5/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 24ms/step - accuracy: 0.9648 - loss: 0.1121 - val_accuracy: 0.7625 - val_loss: 0.7284\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_2_pred_probs = model_2.predict(val_sentences)\n",
        "model_2_pred_probs.shape, model_2_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5VYS32DgL3Nu",
        "outputId": "d6d612a2-41f6-451b-b0e4-a66b91f3df4c"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m24/24\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 26ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((762, 1),\n",
              " array([[0.02239431],\n",
              "        [0.97329104],\n",
              "        [0.9967638 ],\n",
              "        [0.02221144],\n",
              "        [0.00301364],\n",
              "        [0.998816  ],\n",
              "        [0.25711656],\n",
              "        [0.9995484 ],\n",
              "        [0.9987079 ],\n",
              "        [0.14553355]], dtype=float32))"
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_2_preds = tf.squeeze(tf.round(model_2_pred_probs))\n",
        "model_2_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E6onPZP1MJFP",
        "outputId": "79143c7d-b61a-49e4-c992-67c74f0103c8"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([0., 1., 1., 0., 0., 1., 0., 1., 1., 0.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_2_results = calculate_results(val_labels, model_2_preds)\n",
        "model_2_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oMPsN2HaMRW7",
        "outputId": "e54aeea1-ad1f-4b3b-c035-03fb2bb2929d"
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 76.24671916010499,\n",
              " 'precision': 0.7648032381971924,\n",
              " 'recall': 0.7624671916010499,\n",
              " 'f1': 0.7598225942829093}"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "compare_baseline_to_new_results(baseline_results, model_2_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AGtMfBJ9MZnh",
        "outputId": "3a846864-4114-4a59-a898-5ed1b701123c"
      },
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Baseline accuracy:79.27, New accuracy: 76.25, Difference: -3.02\n",
            "Baseline precision:0.81, New precision: 0.76, Difference: -0.05\n",
            "Baseline recall:0.79, New recall: 0.76, Difference: -0.03\n",
            "Baseline f1:0.79, New f1: 0.76, Difference: -0.03\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tf.random.set_seed(42)\n",
        "\n",
        "model_3_embedding = layers.Embedding(input_dim = max_vocab_length,\n",
        "                           output_dim = 128,\n",
        "                           embeddings_initializer = 'uniform',\n",
        "                           input_length = max_length,\n",
        "                           name = 'embedding_3')\n",
        "\n",
        "inputs = layers.Input(shape = (1, ), dtype = 'string')\n",
        "x = text_vectorizer(inputs)\n",
        "x = model_3_embedding(x)\n",
        "x = layers.GRU(64)(x)\n",
        "outputs = layers.Dense(1, activation = 'sigmoid')(x)\n",
        "model_3 = tf.keras.Model(inputs, outputs, name = 'model_3_GRU')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "skIKQBD0Melt",
        "outputId": "81c15b86-8f5d-4ba7-a4b7-5531f05b5330"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/keras/src/layers/core/embedding.py:90: UserWarning: Argument `input_length` is deprecated. Just remove it.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_3.compile(loss = 'binary_crossentropy',\n",
        "                optimizer = 'adam',\n",
        "                metrics = ['accuracy'])"
      ],
      "metadata": {
        "id": "a-TC5PN5Naf5"
      },
      "execution_count": 53,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_3.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 305
        },
        "id": "WeWdr090N027",
        "outputId": "3abef73f-8dbb-42c2-ecf7-fbb1874bfa1f"
      },
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1mModel: \"model_3_GRU\"\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"model_3_GRU\"</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
              "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                        \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape               \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m        Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
              "│ input_layer_2 (\u001b[38;5;33mInputLayer\u001b[0m)           │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)                   │               \u001b[38;5;34m0\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ text_vectorization_1                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m15\u001b[0m)                  │               \u001b[38;5;34m0\u001b[0m │\n",
              "│ (\u001b[38;5;33mTextVectorization\u001b[0m)                  │                             │                 │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ embedding_3 (\u001b[38;5;33mEmbedding\u001b[0m)              │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m15\u001b[0m, \u001b[38;5;34m128\u001b[0m)             │       \u001b[38;5;34m1,280,000\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ gru (\u001b[38;5;33mGRU\u001b[0m)                            │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m64\u001b[0m)                  │          \u001b[38;5;34m37,248\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ dense_2 (\u001b[38;5;33mDense\u001b[0m)                      │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)                   │              \u001b[38;5;34m65\u001b[0m │\n",
              "└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
              "┃<span style=\"font-weight: bold\"> Layer (type)                         </span>┃<span style=\"font-weight: bold\"> Output Shape                </span>┃<span style=\"font-weight: bold\">         Param # </span>┃\n",
              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
              "│ input_layer_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)           │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)                   │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ text_vectorization_1                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">15</span>)                  │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TextVectorization</span>)                  │                             │                 │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ embedding_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)              │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">15</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)             │       <span style=\"color: #00af00; text-decoration-color: #00af00\">1,280,000</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ gru (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GRU</span>)                            │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)                  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">37,248</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ dense_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                      │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)                   │              <span style=\"color: #00af00; text-decoration-color: #00af00\">65</span> │\n",
              "└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m1,317,313\u001b[0m (5.03 MB)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">1,317,313</span> (5.03 MB)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m1,317,313\u001b[0m (5.03 MB)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">1,317,313</span> (5.03 MB)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_3_history = model_3.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs = 5,\n",
        "                              validation_data = (val_sentences, val_labels),\n",
        "                              callbacks = [create_tensorboard_callback(SAVE_DIR, 'GRU')])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pXWgq5JtN2fT",
        "outputId": "2875a6f7-d18d-452e-c804-1f21f38e2ae0"
      },
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/GRU/20241218-140215\n",
            "Epoch 1/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m9s\u001b[0m 27ms/step - accuracy: 0.6557 - loss: 0.6016 - val_accuracy: 0.7756 - val_loss: 0.4590\n",
            "Epoch 2/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m16s\u001b[0m 55ms/step - accuracy: 0.8614 - loss: 0.3361 - val_accuracy: 0.7585 - val_loss: 0.5146\n",
            "Epoch 3/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m17s\u001b[0m 38ms/step - accuracy: 0.9105 - loss: 0.2351 - val_accuracy: 0.7585 - val_loss: 0.5911\n",
            "Epoch 4/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m12s\u001b[0m 56ms/step - accuracy: 0.9395 - loss: 0.1744 - val_accuracy: 0.7690 - val_loss: 0.5863\n",
            "Epoch 5/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m17s\u001b[0m 40ms/step - accuracy: 0.9598 - loss: 0.1356 - val_accuracy: 0.7703 - val_loss: 0.6855\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_3_pred_probs = model_3.predict(val_sentences)\n",
        "model_3_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AAipS84kOKKJ",
        "outputId": "76a00ea2-dc7d-40c9-f559-d37dece9929a"
      },
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m24/24\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 22ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.06221054],\n",
              "       [0.865408  ],\n",
              "       [0.99620605],\n",
              "       [0.11173431],\n",
              "       [0.01833253],\n",
              "       [0.9969473 ],\n",
              "       [0.5050229 ],\n",
              "       [0.99839437],\n",
              "       [0.99701047],\n",
              "       [0.7610251 ]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_3_preds = tf.squeeze(tf.round(model_3_pred_probs))\n",
        "model_3_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P3qp6neaOSM9",
        "outputId": "9f1e92e5-037c-4750-fb39-21e5a3de6bfd"
      },
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([0., 1., 1., 0., 0., 1., 1., 1., 1., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_3_results = calculate_results(val_labels, model_3_preds)\n",
        "model_3_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xUmpzEPUOdQ6",
        "outputId": "401a3334-f2fb-4a83-9c4e-b4fa1813948b"
      },
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 77.03412073490814,\n",
              " 'precision': 0.7729377683268125,\n",
              " 'recall': 0.7703412073490814,\n",
              " 'f1': 0.7677842762403821}"
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "compare_baseline_to_new_results(baseline_results, model_3_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wzJz_uIWOjYa",
        "outputId": "1a52e140-e9af-4399-ee2c-4b12235daff3"
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Baseline accuracy:79.27, New accuracy: 77.03, Difference: -2.23\n",
            "Baseline precision:0.81, New precision: 0.77, Difference: -0.04\n",
            "Baseline recall:0.79, New recall: 0.77, Difference: -0.02\n",
            "Baseline f1:0.79, New f1: 0.77, Difference: -0.02\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tf.random.set_seed(42)\n",
        "\n",
        "model_4_embedding = layers.Embedding(input_dim = max_vocab_length,\n",
        "                                     output_dim = 128,\n",
        "                                     embeddings_initializer = 'uniform',\n",
        "                                     input_length = max_length,\n",
        "                                     name = 'embedding_4')\n",
        "\n",
        "inputs = layers.Input(shape = (1, ), dtype = 'string')\n",
        "x = text_vectorizer(inputs)\n",
        "x = model_4_embedding(x)\n",
        "x = layers.Bidirectional(layers.LSTM(64))(x)\n",
        "outputs = layers.Dense(1, activation = 'sigmoid')(x)\n",
        "model_4 = tf.keras.Model(inputs, outputs, name = 'model_4_Bidirectional')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FATh5D7MOnUJ",
        "outputId": "df69bee5-ae1b-4ec4-bafb-aacd3ee81b98"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/keras/src/layers/core/embedding.py:90: UserWarning: Argument `input_length` is deprecated. Just remove it.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_4.compile(loss = 'binary_crossentropy',\n",
        "                optimizer = 'adam',\n",
        "                metrics = ['accuracy'])"
      ],
      "metadata": {
        "id": "xUKwCX_OPp8m"
      },
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_4.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 305
        },
        "id": "cKkbTOs1P5ZS",
        "outputId": "17b66ace-db25-426f-8945-3665f41a50c3"
      },
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1mModel: \"model_4_Bidirectional\"\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"model_4_Bidirectional\"</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
              "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                        \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape               \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m        Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
              "│ input_layer_3 (\u001b[38;5;33mInputLayer\u001b[0m)           │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)                   │               \u001b[38;5;34m0\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ text_vectorization_1                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m15\u001b[0m)                  │               \u001b[38;5;34m0\u001b[0m │\n",
              "│ (\u001b[38;5;33mTextVectorization\u001b[0m)                  │                             │                 │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ embedding_4 (\u001b[38;5;33mEmbedding\u001b[0m)              │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m15\u001b[0m, \u001b[38;5;34m128\u001b[0m)             │       \u001b[38;5;34m1,280,000\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ bidirectional (\u001b[38;5;33mBidirectional\u001b[0m)        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)                 │          \u001b[38;5;34m98,816\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ dense_3 (\u001b[38;5;33mDense\u001b[0m)                      │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)                   │             \u001b[38;5;34m129\u001b[0m │\n",
              "└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
              "┃<span style=\"font-weight: bold\"> Layer (type)                         </span>┃<span style=\"font-weight: bold\"> Output Shape                </span>┃<span style=\"font-weight: bold\">         Param # </span>┃\n",
              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
              "│ input_layer_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)           │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)                   │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ text_vectorization_1                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">15</span>)                  │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TextVectorization</span>)                  │                             │                 │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ embedding_4 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)              │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">15</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)             │       <span style=\"color: #00af00; text-decoration-color: #00af00\">1,280,000</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ bidirectional (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>)        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)                 │          <span style=\"color: #00af00; text-decoration-color: #00af00\">98,816</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ dense_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                      │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)                   │             <span style=\"color: #00af00; text-decoration-color: #00af00\">129</span> │\n",
              "└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m1,378,945\u001b[0m (5.26 MB)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">1,378,945</span> (5.26 MB)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m1,378,945\u001b[0m (5.26 MB)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">1,378,945</span> (5.26 MB)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_4_history = model_4.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs = 5,\n",
        "                              validation_data = (val_sentences, val_labels),\n",
        "                              callbacks = [create_tensorboard_callback(SAVE_DIR, 'bidirectional_RNN')])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pmEMyRIjP7Y2",
        "outputId": "05be5ed2-89c3-4e71-e0e3-9202bc1d0279"
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/bidirectional_RNN/20241218-140329\n",
            "Epoch 1/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m25s\u001b[0m 77ms/step - accuracy: 0.6777 - loss: 0.5795 - val_accuracy: 0.7743 - val_loss: 0.4640\n",
            "Epoch 2/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m11s\u001b[0m 51ms/step - accuracy: 0.8640 - loss: 0.3283 - val_accuracy: 0.7677 - val_loss: 0.5121\n",
            "Epoch 3/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 70ms/step - accuracy: 0.9156 - loss: 0.2249 - val_accuracy: 0.7585 - val_loss: 0.6009\n",
            "Epoch 4/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m21s\u001b[0m 71ms/step - accuracy: 0.9531 - loss: 0.1417 - val_accuracy: 0.7625 - val_loss: 0.6459\n",
            "Epoch 5/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m20s\u001b[0m 69ms/step - accuracy: 0.9584 - loss: 0.1280 - val_accuracy: 0.7520 - val_loss: 0.7126\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_4_pred_probs = model_4.predict(val_sentences)\n",
        "model_4_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t4yyw15VQQSS",
        "outputId": "50664a65-7df9-4f8f-e3be-71792b5ceba0"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m24/24\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 66ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.16713227],\n",
              "       [0.960741  ],\n",
              "       [0.9991385 ],\n",
              "       [0.02801848],\n",
              "       [0.00812997],\n",
              "       [0.9801316 ],\n",
              "       [0.7089253 ],\n",
              "       [0.999609  ],\n",
              "       [0.99940246],\n",
              "       [0.59869516]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_4_preds = tf.squeeze(tf.round(model_4_pred_probs))\n",
        "model_4_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dVrn2qvHQedN",
        "outputId": "ef689f37-0b7f-4bc9-cde7-e95c98839655"
      },
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([0., 1., 1., 0., 0., 1., 1., 1., 1., 1.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_4_results = calculate_results(val_labels, model_4_preds)\n",
        "model_4_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HcXB6FG7QnJP",
        "outputId": "64e9cc9a-0204-4f65-bda5-1d515c21ab28"
      },
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 75.19685039370079,\n",
              " 'precision': 0.7527859023002904,\n",
              " 'recall': 0.7519685039370079,\n",
              " 'f1': 0.7499245530786587}"
            ]
          },
          "metadata": {},
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "compare_baseline_to_new_results(baseline_results, model_4_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9AbVlF5zQ2lP",
        "outputId": "87868f80-35bf-4881-b2a9-5ddab9162c8d"
      },
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Baseline accuracy:79.27, New accuracy: 75.20, Difference: -4.07\n",
            "Baseline precision:0.81, New precision: 0.75, Difference: -0.06\n",
            "Baseline recall:0.79, New recall: 0.75, Difference: -0.04\n",
            "Baseline f1:0.79, New f1: 0.75, Difference: -0.04\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_test = embedding(text_vectorizer(['this is a test sentence']))\n",
        "conv_1d = layers.Conv1D(filters = 32, kernel_size = 5, activation = 'relu')\n",
        "conv_1d_output = conv_1d(embedding_test)\n",
        "max_pool = layers.GlobalMaxPool1D()\n",
        "max_pool_output = max_pool(conv_1d_output)\n",
        "embedding_test.shape, conv_1d_output.shape, max_pool_output.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uD9_1e-PQ6WX",
        "outputId": "940afa4c-59d9-43f2-ff09-65c319e2da9b"
      },
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(TensorShape([1, 15, 128]), TensorShape([1, 11, 32]), TensorShape([1, 32]))"
            ]
          },
          "metadata": {},
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "  embedding_test[:1], conv_1d_output[:1], max_pool_output[:1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "17UTktrPS-FB",
        "outputId": "95d81dd2-0751-4c62-ef1e-726b49670f4a"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(<tf.Tensor: shape=(1, 15, 128), dtype=float32, numpy=\n",
              " array([[[ 0.01484927,  0.04127872, -0.01587925, ..., -0.01223569,\n",
              "           0.04030773, -0.03100618],\n",
              "         [ 0.01589073,  0.01932128,  0.09458325, ...,  0.06670943,\n",
              "          -0.03487351,  0.00687016],\n",
              "         [ 0.02564662, -0.0163739 , -0.03706221, ...,  0.00995153,\n",
              "           0.01452322, -0.04313828],\n",
              "         ...,\n",
              "         [-0.02927589,  0.04115374,  0.0568382 , ...,  0.03933513,\n",
              "          -0.00237455,  0.01968056],\n",
              "         [-0.02927589,  0.04115374,  0.0568382 , ...,  0.03933513,\n",
              "          -0.00237455,  0.01968056],\n",
              "         [-0.02927589,  0.04115374,  0.0568382 , ...,  0.03933513,\n",
              "          -0.00237455,  0.01968056]]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(1, 11, 32), dtype=float32, numpy=\n",
              " array([[[0.        , 0.        , 0.02493278, 0.0862131 , 0.        ,\n",
              "          0.01275851, 0.09492323, 0.        , 0.        , 0.09402358,\n",
              "          0.        , 0.04717013, 0.04926193, 0.03012793, 0.        ,\n",
              "          0.        , 0.        , 0.        , 0.13843976, 0.08468464,\n",
              "          0.07616352, 0.08290738, 0.        , 0.        , 0.12627687,\n",
              "          0.        , 0.07626042, 0.0809833 , 0.05071025, 0.        ,\n",
              "          0.        , 0.        ],\n",
              "         [0.01062229, 0.05407476, 0.        , 0.        , 0.05805659,\n",
              "          0.        , 0.        , 0.        , 0.        , 0.01533207,\n",
              "          0.        , 0.        , 0.        , 0.13048746, 0.00448363,\n",
              "          0.05846116, 0.        , 0.07373712, 0.0375163 , 0.08976217,\n",
              "          0.1768953 , 0.        , 0.        , 0.01217722, 0.        ,\n",
              "          0.        , 0.04965741, 0.08163109, 0.        , 0.        ,\n",
              "          0.00507716, 0.        ],\n",
              "         [0.        , 0.        , 0.        , 0.        , 0.07230157,\n",
              "          0.00421284, 0.        , 0.        , 0.02242902, 0.        ,\n",
              "          0.01772299, 0.04696259, 0.07261122, 0.        , 0.        ,\n",
              "          0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "          0.12271356, 0.07490239, 0.        , 0.        , 0.12177432,\n",
              "          0.        , 0.04713322, 0.01414745, 0.03371086, 0.01179164,\n",
              "          0.03544812, 0.        ],\n",
              "         [0.02548413, 0.04597379, 0.03992413, 0.        , 0.06601515,\n",
              "          0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "          0.        , 0.05404517, 0.0072039 , 0.09234612, 0.        ,\n",
              "          0.        , 0.01272005, 0.        , 0.        , 0.04512421,\n",
              "          0.08022617, 0.00042066, 0.        , 0.        , 0.        ,\n",
              "          0.        , 0.        , 0.09320663, 0.        , 0.        ,\n",
              "          0.        , 0.07899974],\n",
              "         [0.        , 0.0168229 , 0.        , 0.0043784 , 0.0853263 ,\n",
              "          0.04540216, 0.        , 0.        , 0.        , 0.0115274 ,\n",
              "          0.        , 0.01200638, 0.04663199, 0.0471891 , 0.        ,\n",
              "          0.01977976, 0.        , 0.00029226, 0.        , 0.        ,\n",
              "          0.        , 0.04050409, 0.        , 0.00079576, 0.03262438,\n",
              "          0.        , 0.        , 0.06420843, 0.05082761, 0.04618675,\n",
              "          0.        , 0.10062087],\n",
              "         [0.        , 0.03468408, 0.        , 0.        , 0.06793867,\n",
              "          0.02096266, 0.        , 0.        , 0.        , 0.05911336,\n",
              "          0.        , 0.        , 0.02646207, 0.06003521, 0.01114452,\n",
              "          0.00976776, 0.        , 0.        , 0.        , 0.        ,\n",
              "          0.0014031 , 0.02907735, 0.        , 0.        , 0.05006596,\n",
              "          0.        , 0.        , 0.03451683, 0.08282781, 0.07006238,\n",
              "          0.        , 0.05058802],\n",
              "         [0.        , 0.03468408, 0.        , 0.        , 0.06793867,\n",
              "          0.02096266, 0.        , 0.        , 0.        , 0.05911336,\n",
              "          0.        , 0.        , 0.02646207, 0.06003521, 0.01114452,\n",
              "          0.00976776, 0.        , 0.        , 0.        , 0.        ,\n",
              "          0.0014031 , 0.02907735, 0.        , 0.        , 0.05006596,\n",
              "          0.        , 0.        , 0.03451683, 0.08282781, 0.07006238,\n",
              "          0.        , 0.05058802],\n",
              "         [0.        , 0.03468408, 0.        , 0.        , 0.06793867,\n",
              "          0.02096266, 0.        , 0.        , 0.        , 0.05911336,\n",
              "          0.        , 0.        , 0.02646207, 0.06003521, 0.01114452,\n",
              "          0.00976776, 0.        , 0.        , 0.        , 0.        ,\n",
              "          0.0014031 , 0.02907735, 0.        , 0.        , 0.05006596,\n",
              "          0.        , 0.        , 0.03451683, 0.08282781, 0.07006238,\n",
              "          0.        , 0.05058802],\n",
              "         [0.        , 0.03468408, 0.        , 0.        , 0.06793867,\n",
              "          0.02096266, 0.        , 0.        , 0.        , 0.05911336,\n",
              "          0.        , 0.        , 0.02646207, 0.06003521, 0.01114452,\n",
              "          0.00976776, 0.        , 0.        , 0.        , 0.        ,\n",
              "          0.0014031 , 0.02907735, 0.        , 0.        , 0.05006596,\n",
              "          0.        , 0.        , 0.03451683, 0.08282781, 0.07006238,\n",
              "          0.        , 0.05058802],\n",
              "         [0.        , 0.03468408, 0.        , 0.        , 0.06793867,\n",
              "          0.02096266, 0.        , 0.        , 0.        , 0.05911336,\n",
              "          0.        , 0.        , 0.02646207, 0.06003521, 0.01114452,\n",
              "          0.00976776, 0.        , 0.        , 0.        , 0.        ,\n",
              "          0.0014031 , 0.02907735, 0.        , 0.        , 0.05006596,\n",
              "          0.        , 0.        , 0.03451683, 0.08282781, 0.07006238,\n",
              "          0.        , 0.05058802],\n",
              "         [0.        , 0.03468408, 0.        , 0.        , 0.06793867,\n",
              "          0.02096266, 0.        , 0.        , 0.        , 0.05911336,\n",
              "          0.        , 0.        , 0.02646207, 0.06003521, 0.01114452,\n",
              "          0.00976776, 0.        , 0.        , 0.        , 0.        ,\n",
              "          0.0014031 , 0.02907735, 0.        , 0.        , 0.05006596,\n",
              "          0.        , 0.        , 0.03451683, 0.08282781, 0.07006238,\n",
              "          0.        , 0.05058802]]], dtype=float32)>,\n",
              " <tf.Tensor: shape=(1, 32), dtype=float32, numpy=\n",
              " array([[0.02548413, 0.05407476, 0.03992413, 0.0862131 , 0.0853263 ,\n",
              "         0.04540216, 0.09492323, 0.        , 0.02242902, 0.09402358,\n",
              "         0.01772299, 0.05404517, 0.07261122, 0.13048746, 0.01114452,\n",
              "         0.05846116, 0.01272005, 0.07373712, 0.13843976, 0.08976217,\n",
              "         0.1768953 , 0.08290738, 0.        , 0.01217722, 0.12627687,\n",
              "         0.        , 0.07626042, 0.09320663, 0.08282781, 0.07006238,\n",
              "         0.03544812, 0.10062087]], dtype=float32)>)"
            ]
          },
          "metadata": {},
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tf.random.set_seed(42)\n",
        "\n",
        "model_5_embedding = layers.Embedding(input_dim = max_vocab_length,\n",
        "                                     output_dim = 128,\n",
        "                                     embeddings_initializer = 'uniform',\n",
        "                                     input_length = max_length,\n",
        "                                     name = 'embedding_5')\n",
        "\n",
        "inputs = layers.Input(shape = (1, ), dtype = 'string')\n",
        "x = text_vectorizer(inputs)\n",
        "x = model_5_embedding(x)\n",
        "x = layers.Conv1D(filters = 32, kernel_size = 5, activation = 'relu')(x)\n",
        "x = layers.GlobalMaxPool1D()(x)\n",
        "outputs = layers.Dense(1, activation = 'sigmoid')(x)\n",
        "model_5 = tf.keras.Model(inputs, outputs, name = 'model_5_Conv1D')\n",
        "\n",
        "model_5.compile(loss = 'binary_crossentropy',\n",
        "                optimizer = 'adam',\n",
        "                metrics = ['accuracy'])\n",
        "\n",
        "model_5.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408
        },
        "id": "RKPOmSOdTOn9",
        "outputId": "a32ecec2-e4bb-4ec5-83dd-061274e07d59"
      },
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/keras/src/layers/core/embedding.py:90: UserWarning: Argument `input_length` is deprecated. Just remove it.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1mModel: \"model_5_Conv1D\"\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"model_5_Conv1D\"</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
              "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                        \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape               \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m        Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
              "│ input_layer_4 (\u001b[38;5;33mInputLayer\u001b[0m)           │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)                   │               \u001b[38;5;34m0\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ text_vectorization_1                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m15\u001b[0m)                  │               \u001b[38;5;34m0\u001b[0m │\n",
              "│ (\u001b[38;5;33mTextVectorization\u001b[0m)                  │                             │                 │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ embedding_5 (\u001b[38;5;33mEmbedding\u001b[0m)              │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m15\u001b[0m, \u001b[38;5;34m128\u001b[0m)             │       \u001b[38;5;34m1,280,000\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ conv1d_1 (\u001b[38;5;33mConv1D\u001b[0m)                    │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m11\u001b[0m, \u001b[38;5;34m32\u001b[0m)              │          \u001b[38;5;34m20,512\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ global_max_pooling1d_1               │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m32\u001b[0m)                  │               \u001b[38;5;34m0\u001b[0m │\n",
              "│ (\u001b[38;5;33mGlobalMaxPooling1D\u001b[0m)                 │                             │                 │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ dense_4 (\u001b[38;5;33mDense\u001b[0m)                      │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)                   │              \u001b[38;5;34m33\u001b[0m │\n",
              "└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
              "┃<span style=\"font-weight: bold\"> Layer (type)                         </span>┃<span style=\"font-weight: bold\"> Output Shape                </span>┃<span style=\"font-weight: bold\">         Param # </span>┃\n",
              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
              "│ input_layer_4 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)           │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)                   │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ text_vectorization_1                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">15</span>)                  │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TextVectorization</span>)                  │                             │                 │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ embedding_5 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)              │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">15</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)             │       <span style=\"color: #00af00; text-decoration-color: #00af00\">1,280,000</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ conv1d_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)                    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">11</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)              │          <span style=\"color: #00af00; text-decoration-color: #00af00\">20,512</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ global_max_pooling1d_1               │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)                  │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GlobalMaxPooling1D</span>)                 │                             │                 │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ dense_4 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                      │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)                   │              <span style=\"color: #00af00; text-decoration-color: #00af00\">33</span> │\n",
              "└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m1,300,545\u001b[0m (4.96 MB)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">1,300,545</span> (4.96 MB)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m1,300,545\u001b[0m (4.96 MB)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">1,300,545</span> (4.96 MB)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_5_history = model_5.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs = 5,\n",
        "                              validation_data = (val_sentences, val_labels),\n",
        "                              callbacks = [create_tensorboard_callback(SAVE_DIR, 'Conv1D')])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0NQ86NdwULCF",
        "outputId": "3a783ffe-c0b1-4692-c34a-264a2d77fa8b"
      },
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving TensorBoard log files to: model_logs/Conv1D/20241218-140505\n",
            "Epoch 1/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m12s\u001b[0m 30ms/step - accuracy: 0.6470 - loss: 0.6297 - val_accuracy: 0.7808 - val_loss: 0.4673\n",
            "Epoch 2/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 28ms/step - accuracy: 0.8471 - loss: 0.3750 - val_accuracy: 0.7874 - val_loss: 0.4732\n",
            "Epoch 3/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 18ms/step - accuracy: 0.9085 - loss: 0.2424 - val_accuracy: 0.7900 - val_loss: 0.5262\n",
            "Epoch 4/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m8s\u001b[0m 29ms/step - accuracy: 0.9460 - loss: 0.1584 - val_accuracy: 0.7887 - val_loss: 0.5968\n",
            "Epoch 5/5\n",
            "\u001b[1m215/215\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m11s\u001b[0m 32ms/step - accuracy: 0.9636 - loss: 0.1079 - val_accuracy: 0.7861 - val_loss: 0.6627\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_5_pred_probs = model_5.predict(val_sentences)\n",
        "model_5_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s9IhyhKQUk8v",
        "outputId": "6d4257cc-4acb-4782-b005-95d4087575ed"
      },
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m24/24\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 38ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.6284408 ],\n",
              "       [0.8005239 ],\n",
              "       [0.9995737 ],\n",
              "       [0.07144935],\n",
              "       [0.0040465 ],\n",
              "       [0.99602807],\n",
              "       [0.9799488 ],\n",
              "       [0.9992739 ],\n",
              "       [0.99933875],\n",
              "       [0.09311576]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_5_preds = tf.squeeze(tf.round(model_5_pred_probs))\n",
        "model_5_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SqqKouh7UxTi",
        "outputId": "8daffd23-fc44-48b8-8547-f9628a3eb678"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([1., 1., 1., 0., 0., 1., 1., 1., 1., 0.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_5_results = calculate_results(val_labels, model_5_preds)\n",
        "model_5_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PR7dWEIoU6gq",
        "outputId": "2264495a-026f-46e4-970d-8b444925de1b"
      },
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 78.60892388451444,\n",
              " 'precision': 0.7930934663164497,\n",
              " 'recall': 0.7860892388451444,\n",
              " 'f1': 0.7823860842756168}"
            ]
          },
          "metadata": {},
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "compare_baseline_to_new_results(baseline_results, model_5_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "89u-hMHGVC2H",
        "outputId": "ab2fe8dd-67ea-44c4-cc97-d771fb689de4"
      },
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Baseline accuracy:79.27, New accuracy: 78.61, Difference: -0.66\n",
            "Baseline precision:0.81, New precision: 0.79, Difference: -0.02\n",
            "Baseline recall:0.79, New recall: 0.79, Difference: -0.01\n",
            "Baseline f1:0.79, New f1: 0.78, Difference: -0.00\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow_hub as hub\n",
        "embed = hub.load('https://tfhub.dev/google/universal-sentence-encoder/4')\n",
        "embed_samples = embed([sample_sentence, 'When you call the universal sentence encoder on a sentence, it turns it into numbers.'])\n",
        "print(embed_samples[0][:50])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eHNukwTkVHnu",
        "outputId": "1edb5c04-a855-4834-ca0c-0e54423f76c9"
      },
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tf.Tensor(\n",
            "[-0.01157025  0.02485911  0.02878051 -0.012715    0.03971541  0.08827761\n",
            "  0.02680988  0.05589838 -0.01068731 -0.00597293  0.00639321 -0.01819516\n",
            "  0.00030816  0.09105889  0.05874645 -0.03180629  0.01512474 -0.05162925\n",
            "  0.00991366 -0.06865345 -0.04209306  0.0267898   0.03011009  0.00321065\n",
            " -0.00337968 -0.04787356  0.0226672  -0.00985927 -0.04063615 -0.01292093\n",
            " -0.04666382  0.05630299 -0.03949255  0.00517682  0.02495827 -0.07014439\n",
            "  0.0287151   0.0494768  -0.00633978 -0.08960193  0.02807119 -0.00808364\n",
            " -0.01360601  0.05998649 -0.10361788 -0.05195372  0.00232958 -0.02332531\n",
            " -0.03758106  0.03327729], shape=(50,), dtype=float32)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embed_samples[0].shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mcj8flGqWBBr",
        "outputId": "d56a5082-c422-43a7-c0ae-d06ec9889edd"
      },
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([512])"
            ]
          },
          "metadata": {},
          "execution_count": 77
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence_encoder_layer = hub.KerasLayer('https://tfhub.dev/google/universal-sentence-encoder/4',\n",
        "                                        input_shape = [],\n",
        "                                        dtype = tf.string,\n",
        "                                        trainable = False,\n",
        "                                        name = 'USE')"
      ],
      "metadata": {
        "id": "TuKDNOKtWX7F"
      },
      "execution_count": 78,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import tf_keras\n",
        "\n",
        "model_6 = tf_keras.Sequential([\n",
        "  sentence_encoder_layer,\n",
        "  tf_keras.layers.Dense(64, activation=\"relu\"),\n",
        "  tf_keras.layers.Dense(1, activation=\"sigmoid\")\n",
        "], name=\"model_6_USE\")\n",
        "\n",
        "model_6.compile(loss=\"binary_crossentropy\",\n",
        "                optimizer='adam',\n",
        "                metrics=[\"accuracy\"])\n",
        "\n",
        "model_6.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YTmlEPuZW6Gg",
        "outputId": "1c067f6c-8cd3-4320-9b7a-9786e31ebbcb"
      },
      "execution_count": 79,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_6_USE\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " USE (KerasLayer)            (None, 512)               256797824 \n",
            "                                                                 \n",
            " dense (Dense)               (None, 64)                32832     \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 256830721 (979.73 MB)\n",
            "Trainable params: 32897 (128.50 KB)\n",
            "Non-trainable params: 256797824 (979.61 MB)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_6_history = model_6.fit(train_sentences,\n",
        "                              train_labels,\n",
        "                              epochs = 5,\n",
        "                              validation_data = (val_sentences, val_labels))\n",
        "                              # callbacks=[create_tensorboard_callback(SAVE_DIR,\n",
        "                              #                                        \"tf_hub_sentence_encoder\")])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZrRfCTmpcDz1",
        "outputId": "378bba4d-8db0-47e8-b30e-b9403e580fd2"
      },
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "215/215 [==============================] - 10s 27ms/step - loss: 0.5085 - accuracy: 0.7776 - val_loss: 0.4508 - val_accuracy: 0.7953\n",
            "Epoch 2/5\n",
            "215/215 [==============================] - 5s 22ms/step - loss: 0.4162 - accuracy: 0.8127 - val_loss: 0.4393 - val_accuracy: 0.8084\n",
            "Epoch 3/5\n",
            "215/215 [==============================] - 8s 40ms/step - loss: 0.4018 - accuracy: 0.8203 - val_loss: 0.4349 - val_accuracy: 0.8084\n",
            "Epoch 4/5\n",
            "215/215 [==============================] - 5s 25ms/step - loss: 0.3949 - accuracy: 0.8256 - val_loss: 0.4302 - val_accuracy: 0.8110\n",
            "Epoch 5/5\n",
            "215/215 [==============================] - 5s 22ms/step - loss: 0.3890 - accuracy: 0.8278 - val_loss: 0.4316 - val_accuracy: 0.8123\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_6_pred_probs = model_6.predict(val_sentences)\n",
        "model_6_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SYQWH0uGcvUj",
        "outputId": "b1593fe0-b3a7-4dea-d595-d46f5f81b734"
      },
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 4s 23ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.14426959],\n",
              "       [0.7318657 ],\n",
              "       [0.98428994],\n",
              "       [0.19073291],\n",
              "       [0.7479063 ],\n",
              "       [0.66713846],\n",
              "       [0.97838247],\n",
              "       [0.972514  ],\n",
              "       [0.91854376],\n",
              "       [0.08700102]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_6_preds = tf.squeeze(tf.round(model_6_pred_probs))\n",
        "model_6_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gq8M_puDdcmB",
        "outputId": "2e165ebc-45b5-40c2-bbc1-4718a9af82ce"
      },
      "execution_count": 82,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([0., 1., 1., 0., 1., 1., 1., 1., 1., 0.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_6_results = calculate_results(val_labels, model_6_preds)\n",
        "model_6_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2oHrMVwsdis-",
        "outputId": "e5d04b36-baa9-4b00-8e65-e2d207b6f7df"
      },
      "execution_count": 83,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 81.23359580052494,\n",
              " 'precision': 0.8145687227271431,\n",
              " 'recall': 0.8123359580052494,\n",
              " 'f1': 0.8107894766679798}"
            ]
          },
          "metadata": {},
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "compare_baseline_to_new_results(baseline_results, model_6_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IPNnN54ldr9l",
        "outputId": "3a7c5a50-4361-4084-e0c0-66d2571bd89f"
      },
      "execution_count": 84,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Baseline accuracy:79.27, New accuracy: 81.23, Difference: 1.97\n",
            "Baseline precision:0.81, New precision: 0.81, Difference: 0.00\n",
            "Baseline recall:0.79, New recall: 0.81, Difference: 0.02\n",
            "Baseline f1:0.79, New f1: 0.81, Difference: 0.02\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_sentences_90_percent, train_sentences_10_percent, train_labels_90_percent, train_labels_10_percent = train_test_split(train_sentences,\n",
        "                                                                                                                          train_labels,\n",
        "                                                                                                                          test_size = 0.1,\n",
        "                                                                                                                          random_state = 42)"
      ],
      "metadata": {
        "id": "nEXQM0C3dwFd"
      },
      "execution_count": 85,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(f'Total training examples: {len(train_sentences)}')\n",
        "print(f'Length of 10% training examples: {len(train_sentences_10_percent)}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gLt9Zs-7edJL",
        "outputId": "069188eb-acc9-432d-9cde-f54ab6063f06"
      },
      "execution_count": 86,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total training examples: 6851\n",
            "Length of 10% training examples: 686\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pd.Series(train_labels_10_percent).value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 146
        },
        "id": "7of22G83enyE",
        "outputId": "d687f49e-4fb0-4f2f-b61f-dc70212f82a9"
      },
      "execution_count": 87,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    415\n",
              "1    271\n",
              "Name: count, dtype: int64"
            ],
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>415</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>271</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div><br><label><b>dtype:</b> int64</label>"
            ]
          },
          "metadata": {},
          "execution_count": 87
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_7 = tf_keras.models.clone_model(model_6)\n",
        "\n",
        "model_7.compile(loss = 'binary_crossentropy',\n",
        "                optimizer = 'adam',\n",
        "                metrics = ['accuracy'])\n",
        "\n",
        "model_7.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3SHoQJiffAm1",
        "outputId": "0b3fbbc2-c3d9-4ed7-9e78-2a889ba7f28e"
      },
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_6_USE\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " USE (KerasLayer)            (None, 512)               256797824 \n",
            "                                                                 \n",
            " dense (Dense)               (None, 64)                32832     \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 1)                 65        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 256830721 (979.73 MB)\n",
            "Trainable params: 32897 (128.50 KB)\n",
            "Non-trainable params: 256797824 (979.61 MB)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_7_history = model_7.fit(train_sentences_10_percent,\n",
        "                              train_labels_10_percent,\n",
        "                              epochs = 5,\n",
        "                              validation_data = (val_sentences, val_labels))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RMrbkZOKfZy5",
        "outputId": "8ec67896-bb7f-4b33-dc25-ebc30abe2a02"
      },
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "22/22 [==============================] - 6s 51ms/step - loss: 0.6633 - accuracy: 0.6997 - val_loss: 0.6433 - val_accuracy: 0.6955\n",
            "Epoch 2/5\n",
            "22/22 [==============================] - 1s 27ms/step - loss: 0.5875 - accuracy: 0.8047 - val_loss: 0.5844 - val_accuracy: 0.7388\n",
            "Epoch 3/5\n",
            "22/22 [==============================] - 1s 28ms/step - loss: 0.5100 - accuracy: 0.8265 - val_loss: 0.5359 - val_accuracy: 0.7690\n",
            "Epoch 4/5\n",
            "22/22 [==============================] - 1s 24ms/step - loss: 0.4488 - accuracy: 0.8411 - val_loss: 0.5066 - val_accuracy: 0.7756\n",
            "Epoch 5/5\n",
            "22/22 [==============================] - 1s 24ms/step - loss: 0.4076 - accuracy: 0.8411 - val_loss: 0.4902 - val_accuracy: 0.7782\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_7_pred_probs = model_7.predict(val_sentences)\n",
        "model_7_pred_probs[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CMYFkYOKf855",
        "outputId": "b66a8733-3bf4-40c3-de9c-8fb5d334d6e9"
      },
      "execution_count": 90,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 1s 19ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.24702616],\n",
              "       [0.78049827],\n",
              "       [0.88722175],\n",
              "       [0.28920305],\n",
              "       [0.5622532 ],\n",
              "       [0.8083572 ],\n",
              "       [0.79190683],\n",
              "       [0.82710123],\n",
              "       [0.8281    ],\n",
              "       [0.11172133]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 90
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_7_preds = tf.squeeze(tf.round(model_7_pred_probs))\n",
        "model_7_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Fo-im7V3gKN3",
        "outputId": "65aa1ad7-5d57-4a0d-f4bf-d7c957462db8"
      },
      "execution_count": 91,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([0., 1., 1., 0., 1., 1., 1., 1., 1., 0.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 91
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_7_results = calculate_results(val_labels, model_7_preds)\n",
        "model_7_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fMV7bjOHgWXg",
        "outputId": "256a92c1-e77a-434d-d626-fe53f966c5c2"
      },
      "execution_count": 92,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 77.82152230971128,\n",
              " 'precision': 0.7838273596953228,\n",
              " 'recall': 0.7782152230971129,\n",
              " 'f1': 0.7747045976528383}"
            ]
          },
          "metadata": {},
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "compare_baseline_to_new_results(baseline_results, model_7_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Drre5agjgcIp",
        "outputId": "3baf0b50-e9dc-4bb9-e7b1-749b9bf37f01"
      },
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Baseline accuracy:79.27, New accuracy: 77.82, Difference: -1.44\n",
            "Baseline precision:0.81, New precision: 0.78, Difference: -0.03\n",
            "Baseline recall:0.79, New recall: 0.78, Difference: -0.01\n",
            "Baseline f1:0.79, New f1: 0.77, Difference: -0.01\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "all_model_results = pd.DataFrame({\n",
        "    'baseline' : baseline_results,\n",
        "    'simple_dense' : model_1_results,\n",
        "    'lstm' : model_2_results,\n",
        "    'gru' : model_3_results,\n",
        "    'bidirectional' : model_4_results,\n",
        "    'conv1d' : model_6_results,\n",
        "    'tf_hub_sentence_encoder' : model_6_results,\n",
        "    'tf_hub_10_percent_data' : model_7_results,\n",
        "})\n",
        "\n",
        "all_model_results = all_model_results.transpose()\n",
        "all_model_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "WOIBipWmghQ7",
        "outputId": "fd6bba77-7f37-4b2e-e2ae-3cb5ca1f2a3d"
      },
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                          accuracy  precision    recall        f1\n",
              "baseline                 79.265092   0.811139  0.792651  0.786219\n",
              "simple_dense             78.083990   0.784127  0.780840  0.778263\n",
              "lstm                     76.246719   0.764803  0.762467  0.759823\n",
              "gru                      77.034121   0.772938  0.770341  0.767784\n",
              "bidirectional            75.196850   0.752786  0.751969  0.749925\n",
              "conv1d                   81.233596   0.814569  0.812336  0.810789\n",
              "tf_hub_sentence_encoder  81.233596   0.814569  0.812336  0.810789\n",
              "tf_hub_10_percent_data   77.821522   0.783827  0.778215  0.774705"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4df09097-6040-47cc-9f57-98a673aa88c2\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accuracy</th>\n",
              "      <th>precision</th>\n",
              "      <th>recall</th>\n",
              "      <th>f1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>baseline</th>\n",
              "      <td>79.265092</td>\n",
              "      <td>0.811139</td>\n",
              "      <td>0.792651</td>\n",
              "      <td>0.786219</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>simple_dense</th>\n",
              "      <td>78.083990</td>\n",
              "      <td>0.784127</td>\n",
              "      <td>0.780840</td>\n",
              "      <td>0.778263</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>lstm</th>\n",
              "      <td>76.246719</td>\n",
              "      <td>0.764803</td>\n",
              "      <td>0.762467</td>\n",
              "      <td>0.759823</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>gru</th>\n",
              "      <td>77.034121</td>\n",
              "      <td>0.772938</td>\n",
              "      <td>0.770341</td>\n",
              "      <td>0.767784</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>bidirectional</th>\n",
              "      <td>75.196850</td>\n",
              "      <td>0.752786</td>\n",
              "      <td>0.751969</td>\n",
              "      <td>0.749925</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>conv1d</th>\n",
              "      <td>81.233596</td>\n",
              "      <td>0.814569</td>\n",
              "      <td>0.812336</td>\n",
              "      <td>0.810789</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>tf_hub_sentence_encoder</th>\n",
              "      <td>81.233596</td>\n",
              "      <td>0.814569</td>\n",
              "      <td>0.812336</td>\n",
              "      <td>0.810789</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>tf_hub_10_percent_data</th>\n",
              "      <td>77.821522</td>\n",
              "      <td>0.783827</td>\n",
              "      <td>0.778215</td>\n",
              "      <td>0.774705</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4df09097-6040-47cc-9f57-98a673aa88c2')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-4df09097-6040-47cc-9f57-98a673aa88c2 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-4df09097-6040-47cc-9f57-98a673aa88c2');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-333a9da9-1e03-4ecb-a967-a2a6540bf644\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-333a9da9-1e03-4ecb-a967-a2a6540bf644')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-333a9da9-1e03-4ecb-a967-a2a6540bf644 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_50f66f8a-896a-479b-9f66-9ef8a7bae943\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('all_model_results')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_50f66f8a-896a-479b-9f66-9ef8a7bae943 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('all_model_results');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "all_model_results",
              "summary": "{\n  \"name\": \"all_model_results\",\n  \"rows\": 8,\n  \"fields\": [\n    {\n      \"column\": \"accuracy\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2.1992467484741485,\n        \"min\": 75.19685039370079,\n        \"max\": 81.23359580052494,\n        \"num_unique_values\": 7,\n        \"samples\": [\n          79.26509186351706,\n          78.08398950131233,\n          81.23359580052494\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"precision\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.02384825138410094,\n        \"min\": 0.7527859023002904,\n        \"max\": 0.8145687227271431,\n        \"num_unique_values\": 7,\n        \"samples\": [\n          0.8111390004213173,\n          0.7841274438015813,\n          0.8145687227271431\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"recall\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.021992467484741474,\n        \"min\": 0.7519685039370079,\n        \"max\": 0.8123359580052494,\n        \"num_unique_values\": 7,\n        \"samples\": [\n          0.7926509186351706,\n          0.7808398950131233,\n          0.8123359580052494\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"f1\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.02212380529168578,\n        \"min\": 0.7499245530786587,\n        \"max\": 0.8107894766679798,\n        \"num_unique_values\": 7,\n        \"samples\": [\n          0.7862189758049549,\n          0.7782630349987989,\n          0.8107894766679798\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 94
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "all_model_results['accuracy'] = all_model_results['accuracy'] / 100"
      ],
      "metadata": {
        "id": "3RBOZ4jWhREk"
      },
      "execution_count": 95,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "all_model_results.plot(kind = 'bar').legend(bbox_to_anchor = (1.0, 1.0));"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 593
        },
        "id": "Kb-LHB_Lhgmb",
        "outputId": "4923032d-4970-4493-a6d7-89bf320a2938"
      },
      "execution_count": 96,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAApsAAAJACAYAAADCa62IAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABigklEQVR4nO3deVxUdd//8feAAi6AOy43Si6puKHglpaaJJY/y6U7UxPFpSsLU9Er9VKxzEQt1/SS3FIrU8uyrvQyi6RcyB2sNHfFDdxSAxUU5veHt1MToGIeDsy8no/HPGK+58zM56Dhm+/5Lhar1WoVAAAAYAAXswsAAACA4yJsAgAAwDCETQAAABiGsAkAAADDEDYBAABgGMImAAAADEPYBAAAgGEKmV3AvcjMzNTp06fl6ekpi8VidjkAAOAeWK1W/f7776pYsaJcXOjfclYFImyePn1avr6+ZpcBAADuw4kTJ/Q///M/ZpcBkxSIsOnp6Snp1l9WLy8vk6sBAAD34sqVK/L19bX9Ow7nVCDC5u1b515eXoRNAAAKGIbAOTcGUAAAAMAwhE0AAAAYhrAJAAAAwxSIMZsAAMAxZWZmKj093ewykEuFCxeWq6vrPZ1L2AQAAKZIT0/X0aNHlZmZaXYpuA8lSpRQ+fLl7zoBjLAJAADynNVq1ZkzZ+Tq6ipfX18WfS9ArFarrl69qrNnz0qSKlSocMfzCZsAACDP3bx5U1evXlXFihVVtGhRs8tBLhUpUkSSdPbsWZUrV+6Ot9T5NQIAAOS5jIwMSZKbm5vJleB+3f4l4caNG3c8j7AJAABMw4LvBde9/tkRNgEAAGAYwiYAAAAMwwQhAACQb/iNXJOnn3dsUoc8/TxnRM8mAABAAXa3CTpmI2wCAADkwrp169SyZUuVKFFCpUuX1v/7f/9Phw8fth0/efKkunfvrlKlSqlYsWIKCgrS1q1bbcf/85//qHHjxvLw8FCZMmXUuXNn2zGLxaLVq1fbfV6JEiW0ePFiSdKxY8dksVi0YsUKtWrVSh4eHvroo4904cIFde/eXZUqVVLRokVVr149ffzxx3bvk5mZqSlTpqh69epyd3dX5cqV9dZbb0mSHn/8cYWHh9udf+7cObm5uSkmJuZvfb8ImwAAALmQmpqqiIgI7dixQzExMXJxcVHnzp2VmZmplJQUtWrVSqdOndKXX36phIQEvfbaa7ZdktasWaPOnTvrqaee0u7duxUTE6MmTZrkuoaRI0dq8ODB2rdvn0JCQnT9+nUFBgZqzZo1+vnnn/Xiiy+qV69e2rZtm+01o0aN0qRJkzR27Fjt3btXy5Ytk4+PjySpf//+WrZsmdLS0mznf/jhh6pUqZIef/zxv/X9YswmADii173vcvxy3tSR15z1upGnunbtavd80aJFKlu2rPbu3astW7bo3Llz2r59u0qVKiVJql69uu3ct956S88//7zeeOMNW1uDBg1yXcOQIUPUpUsXu7bhw4fbvh40aJC+/vprrVy5Uk2aNNHvv/+umTNnavbs2erdu7ckqVq1amrZsqUkqUuXLgoPD9cXX3yh5557TpK0ePFi9enT528vT0XYlPjhBKDAudskimMed359vSX1cjy2MurmHV9b+9d9d35zAznrdSN/OXjwoCIjI7V161adP3/e1muZmJio+Ph4NWzY0BY0/yo+Pl4DBgz42zUEBQXZPc/IyNDEiRO1cuVKnTp1Sunp6UpLS7MtvL5v3z6lpaWpbdu22b6fh4eHevXqpUWLFum5557Trl279PPPP+vLL7/827Xe1230OXPmyM/PTx4eHmratKldF212ZsyYoZo1a6pIkSLy9fXV0KFDdf369fsqGAAAwEwdO3bUxYsXNX/+fG3dutU2HjM9Pd22jWNO7nbcYrHIarXatWU3AahYsWJ2z99++23NnDlTI0aM0IYNGxQfH6+QkBClp6ff0+dKt26lf/PNNzp58qTef/99Pf7446pSpcpdX3c3uQ6bK1asUEREhMaNG6ddu3apQYMGCgkJsW3G/lfLli3TyJEjNW7cOO3bt08LFy7UihUr9K9//etvFw8AAJCXLly4oP3792vMmDFq27atateurd9++812vH79+oqPj9fFixezfX39+vXvOOGmbNmyOnPmjO35wYMHdfXq1bvWtXnzZj3zzDN64YUX1KBBA1WtWlUHDhywHa9Ro4aKFClyx8+uV6+egoKCNH/+fC1btkx9+/a96+fei1yHzWnTpmnAgAEKCwuTv7+/oqOjVbRoUS1atCjb87ds2aIWLVqoR48e8vPzU7t27dS9e/e79oYCAADkNyVLllTp0qU1b948HTp0SN99950iIiJsx7t3767y5curU6dO2rx5s44cOaJVq1YpLi5OkjRu3Dh9/PHHtk64n376SZMnT7a9/vHHH9fs2bO1e/du7dixQy+99JIKFy5817pq1Kihb775Rlu2bNG+ffv0j3/8Q8nJybbjHh4eGjFihF577TUtXbpUhw8f1o8//qiFCxfavU///v01adIkWa1Wu1nyf0euwmZ6erp27typ4ODgP97AxUXBwcG2b+JfPfLII9q5c6ctXB45ckRr167VU089lePnpKWl6cqVK3YPAAAAs7m4uGj58uXauXOn6tatq6FDh+rtt9+2HXdzc9P69etVrlw5PfXUU6pXr54mTZokV1dXSVLr1q31ySef6Msvv1RAQIAef/xxuw64qVOnytfXV48++qh69Oih4cOH28Zd3smYMWPUqFEjhYSEqHXr1rbA+2djx47VsGHDFBkZqdq1a6tbt25Z7kx3795dhQoVUvfu3eXhcZdB0PcoVxOEzp8/r4yMDNs0+dt8fHz066+/ZvuaHj166Pz582rZsqWsVqtu3rypl1566Y630aOiouxmaQEAAOdQEHb0CQ4O1t69e+3a/jzOskqVKvr0009zfH2XLl2yzCS/rWLFivr666/t2i5dumT72s/PL8uYTkkqVapUlvU5/8rFxUWjR4/W6NGjczzn/Pnzun79uvr163fH98oNw9fZjI2N1cSJE/Xvf/9bu3bt0meffaY1a9bozTffzPE1o0aN0uXLl22PEydOGF0mAACA07px44aSkpI0ZswYNWvWTI0aNXpg752rns0yZcrI1dXVbgyAJCUnJ6t8+fLZvmbs2LHq1auX+vfvL+nW4NPU1FS9+OKLGj16tFxcsuZdd3d3ubu756Y0AAAA3KfNmzerTZs2evjhh+/YK3s/chU23dzcFBgYqJiYGNs4gMzMTMXExGTZ4ui2q1evZgmUt8ctZNcNbAQj12X7qfdP91MSAABAvtG6dWvDclmuF3WPiIhQ7969FRQUpCZNmmjGjBlKTU1VWFiYJCk0NFSVKlVSVFSUpFtrUU2bNk0NGzZU06ZNdejQIY0dO1YdO3a0hU4AAAA4plyHzW7duuncuXOKjIxUUlKSAgICtG7dOtukocTERLuezDFjxshisWjMmDE6deqUypYtq44dO9o2fgcAAIDjuq/tKsPDw3O8bR4bG2v/AYUKady4cRo3btz9fFS+t69W7TseZ3szAADgzNgb3ZmxJzwAOAZ+niMfI2w6MCZGAQAAsxE2cV8YPgAAeYfOAxRkhi/qDgAAgPsXGxsri8Vit5PQgzg3r9CzCQCAE8t3d6ruNv70gX9e/h/P+sgjj+jMmTPy9r779yY35+YVwibgLJhAAAB5Lj09XW5ubn/rPdzc3HLcqfHvnJtXuI0OOAi/kWvu+Libekvq5fgAANzSunVr2xKQ3t7eKlOmjMaOHWvbfcfPz09vvvmmQkND5eXlpRdffFGStGnTJj366KMqUqSIfH199eqrryo1NdX2vmlpaRoxYoR8fX3l7u6u6tWra+HChZKy3ho/fvy4OnbsqJIlS6pYsWKqU6eO1q5dm+25krRq1SrVqVNH7u7u8vPz09SpU+2uyc/PTxMnTlTfvn3l6empypUra968eQ/se0bYBHBX+2rVvuMDAJzJkiVLVKhQIW3btk0zZ87UtGnTtGDBAtvxd955Rw0aNNDu3bs1duxYHT58WO3bt1fXrl21Z88erVixQps2bbJbszw0NFQff/yxZs2apX379um9995T8eLFs/38V155RWlpafrhhx/0008/afLkyTmeu3PnTj333HN6/vnn9dNPP+n111/X2LFjtXjxYrvzpk6dqqCgIO3evVsvv/yyBg4cqP379//9b5a4jQ5nxO1kAMDf4Ovrq+nTp8tisahmzZr66aefNH36dA0YMECS9Pjjj2vYsGG28/v376+ePXtqyJAhkqQaNWpo1qxZatWqlebOnavExEStXLlS33zzjYKDgyVJVatWzfHzExMT1bVrV9WrV++u506bNk1t27bV2LFjJUkPP/yw9u7dq7ffflt9+vSxnffUU0/p5ZdfliSNGDFC06dP14YNG1SzZs3cf4P+grAJh8MSIQAAIzVr1kwWi8X2vHnz5po6daoyMjIkSUFBQXbnJyQkaM+ePfroo49sbVarVZmZmTp69Kh++uknubq6qlWrVvf0+a+++qoGDhyo9evXKzg4WF27dlX9+vWzPXffvn165pln7NpatGihGTNmKCMjQ66urpJk93qLxaLy5cvr7Nmz91TP3RA2gVzId7M2AQD5TrFixeyep6Sk6B//+IdeffXVLOdWrlxZhw4dytX79+/fXyEhIVqzZo3Wr1+vqKgoTZ06VYMGDbrvmgsXLmz33GKxKDMz877f788YswkAAJALW7dutXv+448/qkaNGrZewr9q1KiR9u7dq+rVq2d5uLm5qV69esrMzNT3339/zzX4+vrqpZde0meffaZhw4Zp/vz52Z5Xu3Ztbd682a5t8+bNevjhh3Os90EjbAIAAORCYmKiIiIitH//fn388cd69913NXjw4BzPHzFihLZs2aLw8HDFx8fr4MGD+uKLL2wThPz8/NS7d2/17dtXq1ev1tGjRxUbG6uVK1dm+35DhgzR119/raNHj2rXrl3asGGDatfO/s7bsGHDFBMTozfffFMHDhzQkiVLNHv2bA0fPvzvfyPuEbfRATg2JoQBBUsB+H8yNDRU165dU5MmTeTq6qrBgwfbljjKTv369fX9999r9OjRevTRR2W1WlWtWjV169bNds7cuXP1r3/9Sy+//LIuXLigypUr61//+le275eRkaFXXnlFJ0+elJeXl9q3b6/p06dne26jRo20cuVKRUZG6s0331SFChU0fvx4u8lBRiNsAijQmBAGIK8VLlxYM2bM0Ny5c7McO3bsWLavady4sdavX5/je3p4eGjatGmaNm1almOtW7e2reMpSe+++26O7/PXcyWpa9eu6tq1a46vya7m+Pj4HM/PLcImAOSACWEA8PcxZhMAAACGoWcTAADgHsXGxppdQoFDzyYAAAAMQ9gEAACAYQibAAAAMAxhEwAAAIYhbAIAAMAwhE0AAAAYhrAJAACQj73++usKCAiwPe/Tp486depkWj25xTqbAAAg37jTFrJGYFta49GzCQAAcJ/S09PNLiHfI2wCAADco9atWys8PFxDhgxRmTJlFBISop9//llPPvmkihcvLh8fH/Xq1Uvnz5+3vSYzM1NTpkxR9erV5e7ursqVK+utt96yHR8xYoQefvhhFS1aVFWrVtXYsWN148YNMy7PEIRNAACAXFiyZInc3Ny0efNmTZo0SY8//rgaNmyoHTt2aN26dUpOTtZzzz1nO3/UqFGaNGmSxo4dq71792rZsmXy8fGxHff09NTixYu1d+9ezZw5U/Pnz9f06dPNuDRDMGYTAAAgF2rUqKEpU6ZIkiZMmKCGDRtq4sSJtuOLFi2Sr6+vDhw4oAoVKmjmzJmaPXu2evfuLUmqVq2aWrZsaTt/zJgxtq/9/Pw0fPhwLV++XK+99loeXZGxCJsAAAC5EBgYaPs6ISFBGzZsUPHixbOcd/jwYV26dElpaWlq27Ztju+3YsUKzZo1S4cPH1ZKSopu3rwpLy8vQ2o3A2ETAAAgF4oVK2b7OiUlRR07dtTkyZOznFehQgUdOXLkju8VFxennj176o033lBISIi8vb21fPlyTZ069YHXbRbCJgAAwH1q1KiRVq1aJT8/PxUqlDVW1ahRQ0WKFFFMTIz69++f5fiWLVtUpUoVjR492tZ2/PhxQ2vOa/c1QWjOnDny8/OTh4eHmjZtqm3btuV4buvWrWWxWLI8OnTocN9FAwAA5AevvPKKLl68qO7du2v79u06fPiwvv76a4WFhSkjI0MeHh4aMWKEXnvtNS1dulSHDx/Wjz/+qIULF0q6FUYTExO1fPlyHT58WLNmzdLnn39u8lU9WLnu2VyxYoUiIiIUHR2tpk2basaMGQoJCdH+/ftVrly5LOd/9tlndmtQXbhwQQ0aNND//u///r3KAQCAwyloi6xXrFhRmzdv1ogRI9SuXTulpaWpSpUqat++vVxcbvXpjR07VoUKFVJkZKROnz6tChUq6KWXXpIkPf300xo6dKjCw8OVlpamDh06aOzYsXr99ddNvKoHK9dhc9q0aRowYIDCwsIkSdHR0VqzZo0WLVqkkSNHZjm/VKlSds+XL1+uokWL3jFspqWlKS0tzfb8ypUruS0TAADggYuNjc3SVqNGDX322Wc5vsbFxUWjR4+2u1X+Z1OmTLHNbr9tyJAhtq9ff/11u/C5ePHi3JRsulzdRk9PT9fOnTsVHBz8xxu4uCg4OFhxcXH39B4LFy7U888/bze49q+ioqLk7e1te/j6+uamTAAAAOQTuQqb58+fV0ZGht1CpJLk4+OjpKSku75+27Zt+vnnn7MdIPtno0aN0uXLl22PEydO5KZMAAAA5BN5Oht94cKFqlevnpo0aXLH89zd3eXu7p5HVQEAAMAouerZLFOmjFxdXZWcnGzXnpycrPLly9/xtampqVq+fLn69euX+yoBAABQIOUqbLq5uSkwMFAxMTG2tszMTMXExKh58+Z3fO0nn3yitLQ0vfDCC/dXKQAAAAqcXN9Gj4iIUO/evRUUFKQmTZpoxowZSk1Ntc1ODw0NVaVKlRQVFWX3uoULF6pTp04qXbr0g6kcAAAA+V6uw2a3bt107tw5RUZGKikpSQEBAVq3bp1t0lBiYqJtXanb9u/fr02bNmn9+vUPpmoAAAAUCPc1QSg8PFzh4eHZHstu/amaNWvKarXez0cBAACgALuv7SoBAACAe0HYBAAAuEdWq1UvvviiSpUqJYvFovj4eLNLyvfydJ1NAACAO9lXq3aefl7tX/fl6vx169Zp8eLFio2NVdWqVXXgwAF17NhRO3fu1JkzZ/T555+rU6dOxhRbQNGzCQAAcI8OHz6sChUq6JFHHlH58uWVmpqqBg0aaM6cOWaXlm/RswkAAHAP+vTpoyVLlkiSLBaLqlSpomPHjunJJ580ubL8jbAJAABwD2bOnKlq1app3rx52r59u1xdXc0uqUAgbAIAANwDb29veXp6ytXV9a7bdOMPjNkEAACAYQibAAAAMAxhEwAAAIZhzCYAAMB9SklJ0aFDh2zPjx49qvj4eJUqVUqVK1c2sbL8g7AJAADyjdwusm62HTt2qE2bNrbnERERkqTevXtr8eLFJlWVvxA2AQAA7tGQIUM0ZMgQ2/PWrVvLarWaV1ABwJhNAAAAGIawCQAAAMMQNgEAAGAYwiYAAAAMQ9gEAACmYXJNwXWvf3aETQAAkOdcXV0lSenp6SZXgvt19epVSVLhwoXveB5LHwEAgDxXqFAhFS1aVOfOnVPhwoXl4kL/V0FhtVp19epVnT17ViVKlLD94pATwiYAAMhzFotFFSpU0NGjR3X8+HGzy8F9KFGihMqXL3/X8wibAADAFG5ubqpRowa30gugwoUL37VH8zbCJgAAMI2Li4s8PDzMLgMGYoAEAAAADEPYBAAAgGEImwAAADAMYRMAAACGIWwCAADAMIRNAAAAGIawCQAAAMMQNgEAAGAYwiYAAAAMc19hc86cOfLz85OHh4eaNm2qbdu23fH8S5cu6ZVXXlGFChXk7u6uhx9+WGvXrr2vggEAAFBw5Hq7yhUrVigiIkLR0dFq2rSpZsyYoZCQEO3fv1/lypXLcn56erqeeOIJlStXTp9++qkqVaqk48ePq0SJEg+ifgAAAORjuQ6b06ZN04ABAxQWFiZJio6O1po1a7Ro0SKNHDkyy/mLFi3SxYsXtWXLFhUuXFiS5Ofnd8fPSEtLU1pamu35lStXclsmAAAA8oFc3UZPT0/Xzp07FRwc/McbuLgoODhYcXFx2b7myy+/VPPmzfXKK6/Ix8dHdevW1cSJE5WRkZHj50RFRcnb29v28PX1zU2ZAAAAyCdyFTbPnz+vjIwM+fj42LX7+PgoKSkp29ccOXJEn376qTIyMrR27VqNHTtWU6dO1YQJE3L8nFGjRuny5cu2x4kTJ3JTJgAAAPKJXN9Gz63MzEyVK1dO8+bNk6urqwIDA3Xq1Cm9/fbbGjduXLavcXd3l7u7u9GlAQAAwGC5CptlypSRq6urkpOT7dqTk5NVvnz5bF9ToUIFFS5cWK6urra22rVrKykpSenp6XJzc7uPsgEAAFAQ5Oo2upubmwIDAxUTE2Nry8zMVExMjJo3b57ta1q0aKFDhw4pMzPT1nbgwAFVqFCBoAkAAODgcr3OZkREhObPn68lS5Zo3759GjhwoFJTU22z00NDQzVq1Cjb+QMHDtTFixc1ePBgHThwQGvWrNHEiRP1yiuvPLirAAAAQL6U6zGb3bp107lz5xQZGamkpCQFBARo3bp1tklDiYmJcnH5I8P6+vrq66+/1tChQ1W/fn1VqlRJgwcP1ogRIx7cVQAAACBfuq8JQuHh4QoPD8/2WGxsbJa25s2b68cff7yfjwIAAEABxt7oAAAAMAxhEwAAAIYhbAIAAMAwhE0AAAAYhrAJAAAAwxA2AQAAYBjCJgAAAAxD2AQAAIBhCJsAAAAwDGETAAAAhiFsAgAAwDCETQAAABiGsAkAAADDEDYBAABgGMImAAAADEPYBAAAgGEImwAAADAMYRMAAACGIWwCAADAMIRNAAAAGIawCQAAAMMQNgEAAGAYwiYAAAAMQ9gEAACAYQibAAAAMAxhEwAAAIYhbAIAAMAwhE0AAAAYhrAJAAAAwxA2AQAAYBjCJgAAAAxD2AQAAIBh7itszpkzR35+fvLw8FDTpk21bdu2HM9dvHixLBaL3cPDw+O+CwYAAEDBkeuwuWLFCkVERGjcuHHatWuXGjRooJCQEJ09ezbH13h5eenMmTO2x/Hjx/9W0QAAACgYch02p02bpgEDBigsLEz+/v6Kjo5W0aJFtWjRohxfY7FYVL58edvDx8fnbxUNAACAgiFXYTM9PV07d+5UcHDwH2/g4qLg4GDFxcXl+LqUlBRVqVJFvr6+euaZZ/TLL7/c8XPS0tJ05coVuwcAAAAKnlyFzfPnzysjIyNLz6SPj4+SkpKyfU3NmjW1aNEiffHFF/rwww+VmZmpRx55RCdPnszxc6KiouTt7W17+Pr65qZMAAAA5BOGz0Zv3ry5QkNDFRAQoFatWumzzz5T2bJl9d577+X4mlGjRuny5cu2x4kTJ4wuEwAAAAYolJuTy5QpI1dXVyUnJ9u1Jycnq3z58vf0HoULF1bDhg116NChHM9xd3eXu7t7bkoDAABAPpSrnk03NzcFBgYqJibG1paZmamYmBg1b978nt4jIyNDP/30kypUqJC7SgEAAFDg5KpnU5IiIiLUu3dvBQUFqUmTJpoxY4ZSU1MVFhYmSQoNDVWlSpUUFRUlSRo/fryaNWum6tWr69KlS3r77bd1/Phx9e/f/8FeCQAAAPKdXIfNbt266dy5c4qMjFRSUpICAgK0bt0626ShxMREubj80WH622+/acCAAUpKSlLJkiUVGBioLVu2yN/f/8FdBQAAAPKlXIdNSQoPD1d4eHi2x2JjY+2eT58+XdOnT7+fjwEAAEABx97oAAAAMAxhEwAAAIYhbAIAAMAwhE0AAAAYhrAJAAAAwxA2AQAAYBjCJgAAAAxD2AQAAIBhCJsAAAAwDGETAAAAhiFsAgAAwDCETQAAABiGsAkAAADDEDYBAABgGMImAAAADEPYBAAAgGEImwAAADAMYRMAAACGIWwCAADAMIRNAAAAGIawCQAAAMMQNgEAAGAYwiYAAAAMQ9gEAACAYQibAAAAMAxhEwAAAIYhbAIAAMAwhE0AAAAYhrAJAAAAwxA2AQAAYBjCJgAAAAxD2AQAAIBh7itszpkzR35+fvLw8FDTpk21bdu2e3rd8uXLZbFY1KlTp/v5WAAAABQwuQ6bK1asUEREhMaNG6ddu3apQYMGCgkJ0dmzZ+/4umPHjmn48OF69NFH77tYAAAAFCy5DpvTpk3TgAEDFBYWJn9/f0VHR6to0aJatGhRjq/JyMhQz5499cYbb6hq1ap/q2AAAAAUHLkKm+np6dq5c6eCg4P/eAMXFwUHBysuLi7H140fP17lypVTv3797ulz0tLSdOXKFbsHAAAACp5chc3z588rIyNDPj4+du0+Pj5KSkrK9jWbNm3SwoULNX/+/Hv+nKioKHl7e9sevr6+uSkTAAAA+YShs9F///139erVS/Pnz1eZMmXu+XWjRo3S5cuXbY8TJ04YWCUAAACMUig3J5cpU0aurq5KTk62a09OTlb58uWznH/48GEdO3ZMHTt2tLVlZmbe+uBChbR//35Vq1Yty+vc3d3l7u6em9IAAACQD+WqZ9PNzU2BgYGKiYmxtWVmZiomJkbNmzfPcn6tWrX0008/KT4+3vZ4+umn1aZNG8XHx3N7HAAAwMHlqmdTkiIiItS7d28FBQWpSZMmmjFjhlJTUxUWFiZJCg0NVaVKlRQVFSUPDw/VrVvX7vUlSpSQpCztAAAAcDy5DpvdunXTuXPnFBkZqaSkJAUEBGjdunW2SUOJiYlycWFjIgAAANxH2JSk8PBwhYeHZ3ssNjb2jq9dvHjx/XwkAAAACiC6IAEAAGAYwiYAAAAMQ9gEAACAYQibAAAAMAxhEwAAAIYhbAIAAMAwhE0AAAAYhrAJAAAAwxA2AQAAYBjCJgAAAAxD2AQAAIBhCJsAAAAwDGETAAAAhiFsAgAAwDCETQAAABiGsAkAAADDEDYBAABgGMImAAAADEPYBAAAgGEImwAAADAMYRMAAACGIWwCAADAMIRNAAAAGIawCQAAAMMQNgEAAGAYwiYAAAAMQ9gEAACAYQibAAAAMAxhEwAAAIYhbAIAAMAwhE0AAAAYhrAJAAAAw9xX2JwzZ478/Pzk4eGhpk2batu2bTme+9lnnykoKEglSpRQsWLFFBAQoA8++OC+CwYAAEDBkeuwuWLFCkVERGjcuHHatWuXGjRooJCQEJ09ezbb80uVKqXRo0crLi5Oe/bsUVhYmMLCwvT111//7eIBAACQv+U6bE6bNk0DBgxQWFiY/P39FR0draJFi2rRokXZnt+6dWt17txZtWvXVrVq1TR48GDVr19fmzZt+tvFAwAAIH/LVdhMT0/Xzp07FRwc/McbuLgoODhYcXFxd3291WpVTEyM9u/fr8ceeyzH89LS0nTlyhW7BwAAAAqeXIXN8+fPKyMjQz4+PnbtPj4+SkpKyvF1ly9fVvHixeXm5qYOHTro3Xff1RNPPJHj+VFRUfL29rY9fH19c1MmAAAA8ok8mY3u6emp+Ph4bd++XW+99ZYiIiIUGxub4/mjRo3S5cuXbY8TJ07kRZkAAAB4wArl5uQyZcrI1dVVycnJdu3JyckqX758jq9zcXFR9erVJUkBAQHat2+foqKi1Lp162zPd3d3l7u7e25KAwAAQD6Uq55NNzc3BQYGKiYmxtaWmZmpmJgYNW/e/J7fJzMzU2lpabn5aAAAABRAuerZlKSIiAj17t1bQUFBatKkiWbMmKHU1FSFhYVJkkJDQ1WpUiVFRUVJujX+MigoSNWqVVNaWprWrl2rDz74QHPnzn2wVwIAAIB8J9dhs1u3bjp37pwiIyOVlJSkgIAArVu3zjZpKDExUS4uf3SYpqam6uWXX9bJkydVpEgR1apVSx9++KG6dev24K4CAAAA+VKuw6YkhYeHKzw8PNtjf534M2HCBE2YMOF+PgYAAAAFHHujAwAAwDCETQAAABiGsAkAAADDEDYBAABgGMImAAAADEPYBAAAgGEImwAAADAMYRMAAACGIWwCAADAMIRNAAAAGIawCQAAAMMQNgEAAGAYwiYAAAAMQ9gEAACAYQibAAAAMAxhEwAAAIYhbAIAAMAwhE0AAAAYhrAJAAAAwxA2AQAAYBjCJgAAAAxD2AQAAIBhCJsAAAAwDGETAAAAhiFsAgAAwDCETQAAABiGsAkAAADDEDYBAABgGMImAAAADEPYBAAAgGEImwAAADAMYRMAAACGIWwCAADAMPcVNufMmSM/Pz95eHioadOm2rZtW47nzp8/X48++qhKliypkiVLKjg4+I7nAwAAwHHkOmyuWLFCERERGjdunHbt2qUGDRooJCREZ8+ezfb82NhYde/eXRs2bFBcXJx8fX3Vrl07nTp16m8XDwAAgPwt12Fz2rRpGjBggMLCwuTv76/o6GgVLVpUixYtyvb8jz76SC+//LICAgJUq1YtLViwQJmZmYqJicnxM9LS0nTlyhW7BwAAAAqeXIXN9PR07dy5U8HBwX+8gYuLgoODFRcXd0/vcfXqVd24cUOlSpXK8ZyoqCh5e3vbHr6+vrkpEwAAAPlErsLm+fPnlZGRIR8fH7t2Hx8fJSUl3dN7jBgxQhUrVrQLrH81atQoXb582fY4ceJEbsoEAABAPlEoLz9s0qRJWr58uWJjY+Xh4ZHjee7u7nJ3d8/DygAAAGCEXIXNMmXKyNXVVcnJyXbtycnJKl++/B1f+84772jSpEn69ttvVb9+/dxXCgAAgAInV7fR3dzcFBgYaDe55/Zkn+bNm+f4uilTpujNN9/UunXrFBQUdP/VAgAAoEDJ9W30iIgI9e7dW0FBQWrSpIlmzJih1NRUhYWFSZJCQ0NVqVIlRUVFSZImT56syMhILVu2TH5+fraxncWLF1fx4sUf4KUAAAAgv8l12OzWrZvOnTunyMhIJSUlKSAgQOvWrbNNGkpMTJSLyx8dpnPnzlV6erqeffZZu/cZN26cXn/99b9XPQAAAPK1+5ogFB4ervDw8GyPxcbG2j0/duzY/XwEAAAAHAB7owMAAMAwhE0AAAAYhrAJAAAAwxA2AQAAYBjCJgAAAAxD2AQAAIBhCJsAAAAwDGETAAAAhiFsAgAAwDCETQAAABiGsAkAAADDEDYBAABgGMImAAAADEPYBAAAgGEImwAAADAMYRMAAACGIWwCAADAMIRNAAAAGIawCQAAAMMQNgEAAGAYwiYAAAAMQ9gEAACAYQibAAAAMAxhEwAAAIYhbAIAAMAwhE0AAAAYhrAJAAAAwxA2AQAAYBjCJgAAAAxD2AQAAIBhCJsAAAAwDGETAAAAhrmvsDlnzhz5+fnJw8NDTZs21bZt23I895dfflHXrl3l5+cni8WiGTNm3G+tAAAAKGByHTZXrFihiIgIjRs3Trt27VKDBg0UEhKis2fPZnv+1atXVbVqVU2aNEnly5f/2wUDAACg4Mh12Jw2bZoGDBigsLAw+fv7Kzo6WkWLFtWiRYuyPb9x48Z6++239fzzz8vd3f2ePiMtLU1XrlyxewAAAKDgyVXYTE9P186dOxUcHPzHG7i4KDg4WHFxcQ+sqKioKHl7e9sevr6+D+y9AQAAkHdyFTbPnz+vjIwM+fj42LX7+PgoKSnpgRU1atQoXb582fY4ceLEA3tvAAAA5J1CZheQHXd393u+5Q4AAID8K1c9m2XKlJGrq6uSk5Pt2pOTk5n8AwAAgCxyFTbd3NwUGBiomJgYW1tmZqZiYmLUvHnzB14cAAAACrZc30aPiIhQ7969FRQUpCZNmmjGjBlKTU1VWFiYJCk0NFSVKlVSVFSUpFuTivbu3Wv7+tSpU4qPj1fx4sVVvXr1B3gpAAAAyG9yHTa7deumc+fOKTIyUklJSQoICNC6detsk4YSExPl4vJHh+np06fVsGFD2/N33nlH77zzjlq1aqXY2Ni/fwUAAADIt+5rglB4eLjCw8OzPfbXAOnn5yer1Xo/HwMAAIACjr3RAQAAYBjCJgAAAAxD2AQAAIBhCJsAAAAwDGETAAAAhiFsAgAAwDCETQAAABiGsAkAAADDEDYBAABgGMImAAAADEPYBAAAgGEImwAAADAMYRMAAACGIWwCAADAMIRNAAAAGIawCQAAAMMQNgEAAGAYwiYAAAAMQ9gEAACAYQibAAAAMAxhEwAAAIYhbAIAAMAwhE0AAAAYhrAJAAAAwxA2AQAAYBjCJgAAAAxD2AQAAIBhCJsAAAAwDGETAAAAhiFsAgAAwDCETQAAABiGsAkAAADD3FfYnDNnjvz8/OTh4aGmTZtq27Ztdzz/k08+Ua1ateTh4aF69epp7dq191UsAAAACpZch80VK1YoIiJC48aN065du9SgQQOFhITo7Nmz2Z6/ZcsWde/eXf369dPu3bvVqVMnderUST///PPfLh4AAAD5W67D5rRp0zRgwACFhYXJ399f0dHRKlq0qBYtWpTt+TNnzlT79u31z3/+U7Vr19abb76pRo0aafbs2X+7eAAAAORvhXJzcnp6unbu3KlRo0bZ2lxcXBQcHKy4uLhsXxMXF6eIiAi7tpCQEK1evTrHz0lLS1NaWprt+eXLlyVJV65cyU25NplpV+94/IrFesfjGdcycjyWkpHzMen+a34QuO7scd3Z47qz4rpzeG+uO88V1Ou+/Vqr9c71wbHlKmyeP39eGRkZ8vHxsWv38fHRr7/+mu1rkpKSsj0/KSkpx8+JiorSG2+8kaXd19c3N+XeM++7nrEvxyNN7vrmd393s3DdOeG6/4rrzunNue78huvOibnX/fvvv8s7H3//YKxchc28MmrUKLve0MzMTF28eFGlS5eWxWLJ01quXLkiX19fnThxQl5eXnn62WbiurluZ8B1c93OwMzrtlqt+v3331WxYsU8/VzkL7kKm2XKlJGrq6uSk5Pt2pOTk1W+fPlsX1O+fPlcnS9J7u7ucnd3t2srUaJEbkp94Ly8vJzqh9NtXLdz4bqdC9ftXMy6bno0kasJQm5ubgoMDFRMTIytLTMzUzExMWrevHm2r2nevLnd+ZL0zTff5Hg+AAAAHEeub6NHRESod+/eCgoKUpMmTTRjxgylpqYqLCxMkhQaGqpKlSopKipKkjR48GC1atVKU6dOVYcOHbR8+XLt2LFD8+bNe7BXAgAAgHwn12GzW7duOnfunCIjI5WUlKSAgACtW7fONgkoMTFRLi5/dJg+8sgjWrZsmcaMGaN//etfqlGjhlavXq26des+uKswkLu7u8aNG5fltr6j47q5bmfAdXPdzsBZrxv5h8XKegQAAAAwCHujAwAAwDCETQAAABiGsAkAAADDEDYBAABgGMImAAAADEPYBAA4nBs3bqhv3746evSo2aUATo+lj3Jw8+ZNxcbG6vDhw+rRo4c8PT11+vRpeXl5qXjx4maXZ4iNGzfqvffe0+HDh/Xpp5+qUqVK+uCDD/TQQw+pZcuWZpeHBygxMfGOxytXrpxHlcAoERER93zutGnTDKzEPN7e3oqPj9dDDz1kdimmuX79utLT0+3anHGrTpgr14u6O4Pjx4+rffv2SkxMVFpamp544gl5enpq8uTJSktLU3R0tNklPnCrVq1Sr1691LNnT+3evVtpaWmSpMuXL2vixIlau3atyRUa5/r163r33Xe1YcMGnT17VpmZmXbHd+3aZVJlxvHz85PFYsnxeEZGRh5WAyPs3r3b7vmuXbt08+ZN1axZU5J04MABubq6KjAw0Izy8kSnTp20evVqDR061OxS8tTVq1f12muvaeXKlbpw4UKW4/z/jbxG2MzG4MGDFRQUpISEBJUuXdrW3rlzZw0YMMDEyowzYcIERUdHKzQ0VMuXL7e1t2jRQhMmTDCxMuP169dP69ev17PPPqsmTZrcMYQ5ir8GkRs3bmj37t2aNm2a3nrrLZOqMk7JkiXv+c/14sWLBleTNzZs2GD7etq0afL09NSSJUtUsmRJSdJvv/2msLAwPfroo2aVaLgaNWpo/Pjx2rx5swIDA1WsWDG746+++qpJlRnrn//8pzZs2KC5c+eqV69emjNnjk6dOqX33ntPkyZNMrs8OCFuo2ejdOnS2rJli2rWrClPT08lJCSoatWqOnbsmPz9/XX16lWzS3zgihYtqr1798rPz8/umo8cOSJ/f39dv37d7BIN4+3trbVr16pFixZml2K6NWvW6O2331ZsbKzZpTxQS5Ysuedze/fubWAl5qhUqZLWr1+vOnXq2LX//PPPateunU6fPm1SZca60+1zi8WiI0eO5GE1eady5cpaunSpWrduLS8vL+3atUvVq1fXBx98oI8//tih71Qhf6JnMxuZmZnZ3mY4efKkPD09TajIeOXLl9ehQ4fk5+dn175p0yZVrVrVnKLySKVKlRz2zzW3atasqe3bt5tdxgPniAEyN65cuaJz585laT937px+//13EyrKG846OejixYu2n9teXl623vqWLVtq4MCBZpYGJ8Vs9Gy0a9dOM2bMsD23WCxKSUnRuHHj9NRTT5lXmIEGDBigwYMHa+vWrbJYLDp9+rQ++ugjDR8+3OF/OE2dOlUjRozQ8ePHzS4lz1y5csXucfnyZf36668aM2aMatSoYXZ5eeb69etZvheOqHPnzgoLC9Nnn32mkydP6uTJk1q1apX69eunLl26mF2e4dLT07V//37dvHnT7FLyRNWqVW1Bu1atWlq5cqUk6T//+Y9KlChhYmVwWlZkceLECau/v7+1du3a1kKFClmbNWtmLV26tLVmzZrW5ORks8szRGZmpnXChAnWYsWKWS0Wi9VisVg9PDysY8aMMbs0w509e9baunVrq4uLi7V48eLWkiVL2j0ckcVisbq4uNg9LBaLtXLlytYtW7aYXZ6hUlJSrK+88oq1bNmyWb4HLi4uZpdniNTUVOvAgQOt7u7utut0c3OzDhw40JqSkmJ2eYZJTU219u3b1+rq6mp1dXW1Hj582Gq1Wq3h4eHWqKgok6szzrRp06wzZ860Wq1W6zfffGP18PCw/dnPmDHD5OrgjBizmYObN29q+fLl2rNnj1JSUtSoUSP17NlTRYoUMbs0Q6Wnp+vQoUNKSUmRv7+/wy7z9GfBwcFKTExUv3795OPjk2UiiSPegv3+++/tnru4uKhs2bKqXr26ChVy7NE1r7zyijZs2KA333wz28kTPXv2NLtEw6Smpurw4cOSpGrVqmWZMONoBg8erM2bN2vGjBlq37699uzZo6pVq+qLL77Q66+/nmWinKM6fvy4du7cqerVq6t+/fpmlwMnRNhEtq5cuaLvvvtONWvWVO3atc0ux1BFixZVXFycGjRoYHYpeeLGjRv6xz/+obFjxzrl+oNMnnAeVapU0YoVK9SsWTO7iY+HDh1So0aNHHbYxNKlS9WtWze5u7vbtaenp2v58uUKDQ01qTI4K8fuwvgbDh48mOO6i5GRkSZVZZznnntOjz32mMLDw3Xt2jU1btxYR48eldVq1fLly9W1a1ezSzRMrVq1dO3aNbPLyDOFCxfWqlWrNHbsWLNLMQWTJ/5w+PBhDRgwQN99953ZpRji3LlzKleuXJb21NRUh17iLCwsTO3bt89y7b///rvCwsIIm8hzTBDKxvz581W7dm1FRkbq008/1eeff257rF692uzyDPHDDz/Y1tv7/PPPlZmZqUuXLmnWrFkOv87mpEmTNGzYMMXGxurChQtOMWHk9mLXzojJE39ISUnJMqTCkQQFBWnNmjW257cD5oIFC9S8eXOzyjKc1WrNNkyfPHlS3t7eJlQEZ0fPZjYmTJigt956SyNGjDC7lDxz+fJllSpVSpK0bt06de3aVUWLFlWHDh30z3/+0+TqjNW+fXtJUtu2be3ab//AdsTdNpx1sWvpVq9PQkKCWrVqpZEjR6pjx46aPXu2bty44XDbNs6aNeuOx0+dOpVHlZhj4sSJevLJJ7V3717dvHlTM2fO1N69e7VlyxaHDNkNGzaUxWKRxWJR27Zt7cZfZ2Rk6OjRo7afd0BeYsxmNry8vBQfH+/w60v+2cMPP6wJEyaoQ4cOeuihh7R8+XI9/vjjSkhIUNu2bXX+/HmzSzTM3f7RadWqVR5VknecdbHr7Djy5AkXFxdVqFBBbm5u2R5PT09XUlKSQ/5Cddvhw4c1adIkJSQk2CZ7jhgxQvXq1TO7tAfujTfesP132LBhdhM83dzc5Ofnp65du+b49wEwCmEzG/369VPjxo310ksvmV1Knvn3v/+twYMHq3jx4qpSpYp27dolFxcXvfvuu/rss8/str5zNImJifL19c1y28lqterEiROqXLmySZUBf89DDz2kyZMn67nnnsv2eHx8vAIDAx06bDqjJUuWqFu3bvLw8DC7FEASYTNbUVFRmjZtmjp06KB69eqpcOHCdscd9Rbjjh07dOLECT3xxBO234jXrFmjEiVKOPRWjq6urjpz5kyWwfQXLlxQuXLlHPIf4oiIiGzbLRaLPDw8VL16dT3zzDO2oRWOJiYmRjExMdlOAFy0aJFJVT14zz77rKpVq6bJkydnezwhIUENGzbM8j0oyHIzztrLy8vASgDcRtjMBrcYnYuLi4uSk5NVtmxZu/bjx4/L399fqampJlVmnDZt2mjXrl3KyMhQzZo1JUkHDhyQq6uratWqpf3798tisWjTpk3y9/c3udoH64033tD48eMVFBSkChUqZOnR/vzzz02q7MHbu3evrl69qqCgoGyP37hxQ6dPn1aVKlXyuDLjuLi43PNMc0f8RVK6dV3Tp0/XypUrlZiYqPT0dLvjt1dgAPIKE4Sy4Yz76WZkZGjx4sU59vY44tIot3v3LBaLxo4dq6JFi9qOZWRkaOvWrQoICDCpOmPd7rV8//33bb07ly9fVv/+/dWyZUsNGDBAPXr00NChQ/X111+bXO2DFR0drcWLF6tXr15ml2K4u/2iULhwYYcKmpLshvwcO3ZMI0eOVJ8+fWyzz+Pi4rRkyRJFRUWZVaLh3njjDS1YsEDDhg3TmDFjNHr0aB07dkyrV692yKX7kP/RswlJUnh4uBYvXqwOHTpk29szffp0kyozTps2bSTdmiDUvHlzu0HztwfTDx8+3CH3Cq9UqZK++eabLGHkl19+Ubt27XTq1Cnt2rVL7dq1c7jJYaVLl9a2bdtUrVo1s0vJMxMmTFDPnj2dbhH/tm3bqn///urevbtd+7JlyzRv3jzFxsaaU5jBqlWrplmzZqlDhw7y9PRUfHy8re3HH3/UsmXLzC4RToaw+X8iIiL05ptvqlixYjmOZ7vN0ZZHkaQyZcpo6dKleuqpp8wuJc+FhYVp5syZTjV+q3jx4vrqq6/UunVru/bY2Fh17NhRv//+u44cOaKAgACHW2t0xIgRKl68uFMtat+gQQP9/PPPatq0qV544QU999xzKlOmjNllGa5o0aJKSEjI8gvjgQMHFBAQoKtXr5pUmbGKFSumffv2qXLlyqpQoYLWrFmjRo0a6ciRI2rYsKEuX75sdolwMtxG/z+7d+/WjRs3bF/nxFF3nXBzc1P16tXNLsMU77//vt3z21t11qpVS7Vq1TKpKmM988wz6tu3r6ZOnarGjRtLkrZv367hw4erU6dOkqRt27bp4YcfNrFKY1y/fl3z5s3Tt99+q/r162eZAOiIv0wmJCTol19+0UcffaR33nlHQ4YM0RNPPKGePXuqU6dOdkNIHImvr6/mz5+vKVOm2LUvWLBAvr6+JlVlvP/5n//RmTNnVLlyZVWrVk3r169Xo0aNtH379ixbWAJ5gZ5NSJKmTp2qI0eOaPbs2Q4bqHPy1606GzRooGPHjjn0Vp0pKSkaOnSoli5dqps3b0qSChUqpN69e2v69OkqVqyY4uPjJcnhxq3eHj6RHYvF4pDjk/9q8+bNWrZsmT755BNdv37d4Xqvb1u7dq26du2q6tWrq2nTppJu/RJ18OBBrVq1ymHv5IwcOVJeXl7617/+pRUrVuiFF16Qn5+fEhMTNXToUE2aNMnsEuFkCJuQJHXu3FkbNmxQqVKlVKdOnSy9PZ999plJlRmvfPny+vrrr9WgQQMtW7ZM48aNU0JCgpYsWaJ58+bdsae7oEtJSbGtrlC1alW7RaDhuOLj4/Xhhx9q+fLlunDhgq5du2Z2SYY5efKk/v3vf+vXX3+VJNWuXVsvvfSSQ/ds/lVcXJzi4uJUo0YNdezY0exy4IQIm/+nS5cu93yuIwavsLCwOx7/661mR1KkSBEdOHBAvr6+Cg0NVcWKFTVp0iQlJibK399fKSkpZpcIg5w8eVLSrduOju7o0aNatmyZli1bpv3796tVq1bq0aOHnn32WfbLBmAoxmz+H2f/YevIYfJufH19FRcXp1KlSmndunVavny5JOm3335jBw4HlJmZqQkTJmjq1Km2XyQ8PT01bNgwjR49Wi4uLiZX+OA1a9ZM27dvV/369RUWFqbu3burUqVKZpeVJy5duqSFCxdq3759kqQ6deqob9++Dvcz/8svv7znc59++mkDKwGyomcTNjdv3lRsbKwOHz6sHj16yNPTU6dPn5aXl5dD317981adlStX1u7du51mq05nNGrUKC1cuFBvvPGGbWesTZs26fXXX9eAAQP01ltvmVzhgzd69Gj17NnT4Rbov5sdO3YoJCRERYoUUZMmTSTdmgh37do126QZR/HXX5IsFov++s/77fH4jrqYPfIvwmYOnC14HT9+XO3bt1diYqLS0tJ04MABVa1aVYMHD1ZaWpqio6PNLtFQO3fuVGJiotq1a6dixYpJurVVZ8mSJfXII4+YXB0epIoVKyo6OjpL784XX3yhl19+WadOnTKpMjxojz76qKpXr6758+erUKFbN/Ju3ryp/v3768iRI/rhhx9MrtAY3377rUaMGKGJEyfaLWY/ZswYTZw4UU888YTJFcLZEDaz4YzBq1OnTvL09NTChQtVunRpJSQkqGrVqoqNjdWAAQN08OBBs0t8oO62luqfOeJSOM7Mw8NDe/bsybKs0/79+xUQEOCQk2WccYcw6dZ47N27d2dZwmzv3r0KCgpy2HU269atq+joaLVs2dKufePGjXrxxRdtQwqAvMKYzWwMHjxYQUFBSkhIUOnSpW3tnTt31oABA0yszDgbN27Uli1b7HbRkSQ/Pz+H7Om51xnmzrYMlDNo0KCBZs+erVmzZtm1z549Ww0aNDCpKmMNHjzYtkNY3bp1nebvtZeXlxITE7OEzRMnTsjT09Okqox3+PBhlShRIku7t7e3jh07luf1AITNbDhb8JJuTZrIbhzPyZMnHfKHMuMwndeUKVPUoUMHffvtt3a3GE+cOKG1a9eaXJ0xli9frpUrVzrsupI56datm/r166d33nnHNhxm8+bN+uc//5llC0tH0rhxY0VEROiDDz6Qj4+PJCk5OVn//Oc/bWNXgbxE2MyGswUvSWrXrp1mzJihefPmSbrVo5eSkqJx48Y53T9QcGytWrXSgQMHNGfOHNvai126dNHLL7+sihUrmlydMZx1h7B33nlHFotFoaGhts0LChcurIEDBzr0wuaLFi1S586dVblyZdt6oidOnFCNGjW0evVqc4uDU2LMZja6desmb29vzZs3T56entqzZ4/Kli2rZ555RpUrV3bIZYJOnjypkJAQWa1WHTx4UEFBQTp48KDKlCmjH374QeXKlTO7RAD3yZl3CJOkq1ev6vDhw5KkatWqOez2nH9mtVr1zTff2C1mHxwc7JR//jAfYTMbzhq8bt68qeXLl2vPnj1KSUlRo0aN1LNnTxUpUsTs0oC/Zc+ePapbt65cXFy0Z8+eO55bv379PKoq7zjrDmGXL19WRkaGSpUqZdd+8eJFFSpUSF5eXiZVlj/Uq1dPa9eudardlGAOwmYObt68qRUrVighIYHgBRRwLi4uSkpKUrly5eTi4pLtGoTSreEjjrgGobPuEPbkk0+qY8eOevnll+3ao6Oj9eWXXzrsGN175enpaVt5BDASYdOJseMEnMXx48dVuXJlWSwWHT9+/I7nVqlSJY+qgtFKlSqlzZs3q3bt2nbtv/76q1q0aKELFy6YVFn+QNhEXmGCUDaWLFmiMmXKqEOHDpKk1157TfPmzZO/v78+/vhjh/nHqFOnTnbP2XECjurP/88eP35cjzzyiG2R79tu3rypLVu2OMz/39k5d+6c9u/fL0mqWbOmypYta3JFxkpLS7NNDPqzGzduOOR6qkB+5XibAD8AEydOtN0uj4uL0+zZszVlyhSVKVNGQ4cONbm6ByczM9P2WL9+vQICAvTf//5Xly5d0qVLl/Tf//5XjRo10rp168wuFXhg2rRpo4sXL2Zpv3z5stq0aWNCRcZLTU1V3759VaFCBT322GN67LHHVLFiRfXr189hFzaXpCZNmthW2Piz6OhoBQYGmlAR4Jzo2czGiRMnbMuErF69Ws8++6xefPFFtWjRQq1btza3OIMMGTIky44TISEhKlq0KDtOwKFYrdZsZ+ReuHDBtlWpo4mIiND333+v//znP3b7wb/66qsaNmyY5s6da3KFxpgwYYKCg4OVkJCgtm3bSpJiYmK0fft2rV+/3uTqAOdB2MxG8eLFdeHCBVWuXFnr16+3bW3o4eHhsLde2HECjq5Lly6Sbg0N6dOnj9zd3W3HMjIytGfPHtvC345m1apV+vTTT+1+WX7qqadUpEgRPffccw4bNlu0aKG4uDi9/fbbWrlypYoUKaL69etr4cKFqlGjhtnlAU6DsJmNJ554Qv3791fDhg114MAB26Lmv/zyi/z8/MwtziDsOAFH5+3tLelWz6anp6fdyhJubm5q1qyZw25He/XqVdv/139Wrlw5h76NLkkBAQH66KOPzC4jTy1dulTdunWz+4VKktLT07V8+XKFhoZKkt57771s/14ADxqz0bNx6dIljRkzRidOnNDAgQPVvn17SdK4cePk5uam0aNHm1zhg3fo0CF17txZBw4cyHbHCWfcfQSO6Y033tA///lPp1jY+7a2bduqdOnSWrp0qTw8PCRJ165dU+/evXXx4kV9++23JldonMzMTB06dEhnz55VZmam3bHHHnvMpKqM5erqqjNnzmRZE/rChQsqV64cEz6R5wibsGHHCTiDo0eP6ubNm1luox48eFCFCxd2yLsXP/30k9q3b6+0tDQ1aNBAkpSQkCB3d3etX79ederUMblCY/z444/q0aOHjh8/nu1KG44aulxcXJScnJxltYGEhIQcJ8gBRiJs3sHVq1eVmJio9PR0u3ZH3GHkXrHjBAq6Vq1aqW/fvurdu7dd+4cffqgFCxYoNjbWnMIMdvXqVX300Ud2v0w6+kYVAQEBevjhh/XGG2+oQoUKWX5xvj20wlE0bNhQFotFCQkJqlOnjt3yXhkZGTp69Kjat2+vlStXmlglnBFjNrNx7tw59enTJ8clfxz1t+F7cezYMd24ccPsMoD7tnv3btuM7D9r1qyZwsPDTajIeFFRUfLx8ckyJnXRokU6d+6cRowYYVJlxjp48KA+/fRTpxkGdHvt5Pj4eIWEhKh48eK2Y25ubvLz81PXrl1Nqg7OjLCZjSFDhujy5cvaunWrWrdurc8//1zJycmaMGGCpk6danZ5AP4Gi8Wi33//PUv77X20HdF7772nZcuWZWmvU6eOnn/+eYcNm02bNtWhQ4ecJmyOGzdOkuTn56du3brZxucCZiNsZuO7777TF198oaCgILm4uKhKlSp64okn5OXlpaioKNvOQgAKnscee0xRUVH6+OOP5erqKunW3YqoqCi7dWYdSVJSkipUqJClvWzZsjpz5owJFeWNQYMGadiwYUpKSlK9evVUuHBhu+OOOiTq9hCR9PT0bCdGVa5c2Yyy4MQIm9lITU21zeIrWbKkzp07p4cfflj16tXTrl27TK4OwN8xefJkPfbYY6pZs6YeffRRSdLGjRt15coVfffddyZXZwxfX19t3rxZDz30kF375s2bVbFiRZOqMt7tW8Z9+/a1td3elteRJwgdPHhQffv21ZYtW+zaHf26kX8RNrNRs2ZN7d+/X35+fmrQoIHee+89+fn5KTo6OtveAQAFh7+/v/bs2aPZs2crISFBRYoUUWhoqMLDw1WqVCmzyzPEgAEDNGTIEN24cUOPP/64pFs76bz22msaNmyYydUZ5+jRo2aXYIo+ffqoUKFC+uqrr7KdGAXkNWajZ+PDDz/UzZs31adPH+3cuVPt27fXhQsX5ObmpiVLlqhbt25ml2gaT09PJSQkqGrVqmaXAuAeWa1WjRw5UrNmzbKtruHh4aERI0YoMjLS5OrwoBUrVkw7d+5UrVq1zC4FkETYvCur1apr167p119/VeXKlVWmTBmzSzLc9evXcxxYvmzZMj3zzDMOu4c0nMPGjRv13nvv6ciRI/rkk09UqVIlffDBB3rooYccdtymJKWkpGjfvn0qUqSIatSokWWHGUf0wQcfKDo6WkePHlVcXJyqVKmiGTNm6KGHHtIzzzxjdnmGaNy4saZPn+7Qf5dRsLiYXUB+tXDhQtWtW1ceHh4qWbKkQkNDtXr1arPLMkxmZqbefPNNVapUScWLF9eRI0ckSWPHjtXChQtt5/Xo0YOgiQJt1apVCgkJUZEiRbRr1y6lpaVJujUbfeLEiSZXZ6zixYurcePGqlu3rlMEzblz5yoiIkJPPfWULl26ZBurWKJECc2YMcPc4gw0efJkvfbaa4qNjdWFCxd05coVuweQ1wib2YiMjNTgwYPVsWNHffLJJ/rkk0/UsWNHDR061GFvOU2YMEGLFy/WlClT5ObmZmuvW7euFixYYGJlwIM1YcIERUdHa/78+Xazk1u0aMEEQAfz7rvvav78+Ro9erRt5QFJCgoK0k8//WRiZcYKDg7Wjz/+qLZt26pcuXIqWbKkSpYsqRIlSqhkyZJmlwcnxAShbMydO1fz589X9+7dbW1PP/206tevr0GDBmn8+PEmVmeMpUuXat68eWrbtq1eeuklW3uDBg1sO44AjmD//v3Z7ont7e2tS5cu5X1BMMzRo0fVsGHDLO3u7u5KTU01oaK8sWHDBrNLAOwQNrNx48YNBQUFZWkPDAzUzZs3TajIeKdOncp24ePMzEx2DIJDKV++vA4dOpRlD/RNmzYx8c3BPPTQQ4qPj1eVKlXs2tetW6fatWubVJXxWrVqZXYJgB1uo2ejV69emjt3bpb2efPmqWfPniZUZDx/f39t3LgxS/unn36abc8AUFANGDBAgwcP1tatW2WxWHT69Gl99NFHGj58uAYOHGh2eXiAIiIi9Morr2jFihWyWq3atm2b3nrrLY0aNUqvvfaa2eUZauPGjXrhhRf0yCOP6NSpU5JuTZbatGmTyZXBGdGz+X8iIiJsX1ssFi1YsEDr169Xs2bNJElbt25VYmKiQkNDzSrRUJGRkerdu7dOnTqlzMxMffbZZ9q/f7+WLl2qr776yuzygAdm5MiRyszMVNu2bXX16lU99thjcnd31/DhwzVo0CCzy8MD1L9/fxUpUkRjxozR1atX1aNHD1WsWFEzZ87U888/b3Z5hlm1apV69eqlnj17ZjsJbu3atSZXCGfD0kf/p02bNvd0nsVicdhdRjZu3Kjx48crISFBKSkpatSokSIjI9WuXTuzSwMeiIyMDG3evFn169dX0aJFdejQIaWkpMjf31/Fixc3uzwY6OrVq0pJSbHtDvdnmzdvVlBQkMPM0G/YsKGGDh2q0NBQu7WRd+/erSeffFJJSUlmlwgnQ9gE4FQ8PDy0b9++LFs3wnl5eXkpPj7eYcbsFi1aVHv37pWfn59d2Dxy5Ij8/f11/fp1s0uEk2HMJgCnUrduXds6soB0a/MOR3J7EtxfMQkOZmHMphMrWbLkPe+Ze/HiRYOrAfLGhAkTNHz4cL355psKDAzMskmBl5eXSZUBD8btSXCLFi2yTYKLi4vT8OHDNXbsWLPLgxMibDoxR95BA8jJU089JenW2rl//mXLarXKYrHYdpkBCiomwSG/YcwmAKfy/fff3/E4axQ6nz+Pa3Qk6enpTIJDvkDYhE1GRoY+//xz7du3T9KttTefeeYZFSpEBzgAx+VoE4QuX76sjIwMlSpVyq794sWLKlSoEENFkOdIEZAk/fLLL3r66aeVlJSkmjVrSpImT56ssmXL6j//+Y/q1q1rcoXA/duzZ4/q1q0rFxcX7dmz547n1q9fP4+qQn7haH0uzz//vDp27KiXX37Zrn3lypX68ssvWWcTeY6eTUiSmjdvrrJly2rJkiUqWbKkJOm3335Tnz59dO7cOW3ZssXkCoH75+LioqSkJJUrV04uLi6yWCzZBgzGbDqemzdvKjY2VocPH1aPHj3k6emp06dPy8vLy2FvK5cqVUqbN2/OsiXnr7/+qhYtWujChQsmVQZnRc8mJEnx8fHasWOHLWhKt2arv/XWW2rcuLGJlQF/39GjR1W2bFnb13AOx48fV/v27ZWYmKi0tDQ98cQT8vT01OTJk5WWlqbo6GizSzREWlqabt68maX9xo0bunbtmgkVwdmxziYkSQ8//LCSk5OztJ89e1bVq1c3oSLgwalSpYpt5nmVKlXu+IDjGDx4sIKCgvTbb7+pSJEitvbOnTsrJibGxMqM1aRJE82bNy9Le3R0tAIDA02oCM6Onk1IkqKiovTqq6/q9ddft+0H/+OPP2r8+PGaPHmyrly5YjuXweUoaL788st7Pvfpp582sBLkpY0bN2rLli1yc3Oza/fz89OpU6dMqsp4EyZMUHBwsBISEtS2bVtJUkxMjLZv367169ebXB2cEWM2IenWmLbbbvcA3f6r8efnjGlDQfTnv9+SsozZ/PN6m/z9dhwlS5bU5s2b5e/vb7e80aZNm9S1a9ds7+Y4ioSEBE2ZMkXx8fEqUqSI6tevr1GjRqlGjRpmlwYnRM8mJEkbNmwwuwTAMJmZmbavv/32W40YMUITJ05U8+bNJUlxcXEaM2aMJk6caFaJMEC7du00Y8YM2y1li8WilJQUjRs3zra4v6O5ceOG/vGPf2js2LH66KOPzC4HkETPJgAnU7duXUVHR6tly5Z27Rs3btSLL75oW2cWBd/JkycVEhIiq9WqgwcPKigoSAcPHlSZMmX0ww8/qFy5cmaXaAhvb2/Fx8froYceMrsUQBJhE39y/fp17dmzR2fPnrXrCZIYxwbHUaRIEW3fvj3L2rF79uxR06ZNma3rYG7evKkVK1YoISFBKSkpatSokXr27Gk3YcjR9O7dWwEBARo6dKjZpQCSCJv4P+vWrVNoaKjOnz+f5RjjNOFIHnvsMXl4eOiDDz6Qj4+PJCk5OVmhoaG6fv36XbezBPK7CRMmaOrUqWrbtq0CAwNVrFgxu+OvvvqqSZXBWRE2IUmqUaOG2rVrp8jISNs/wIAjOnTokDp37qwDBw7I19dXknTixAnVqFFDq1evZqkvBxIVFSUfHx/17dvXrn3RokU6d+6cRowYYVJlxrrT7XOLxaIjR47kYTUAYRP/x8vLS7t371a1atXMLgUwnNVq1TfffKNff/1VklS7dm0FBwfbzUpHwefn56dly5bpkUcesWvfunWrnn/+eRb4B/IIs9EhSXr22WcVGxtL2IRTsFgsateundq1a2d2KTBQUlKSKlSokKW9bNmyOnPmjAkV5a309HQdPXpU1apVU6FC/HMP8/C3D5Kk2bNn63//93+1ceNG1atXT4ULF7Y7zhgfFGSzZs3Siy++KA8PD82aNeuO5/J33XH4+vpq8+bNWW4rb968WRUrVjSpKuNdvXpVgwYN0pIlSyRJBw4cUNWqVTVo0CBVqlRJI0eONLlCOBtuo0OStHDhQr300kvy8PBQ6dKl7W4nMsYHBd1DDz2kHTt2qHTp0oxncyJTpkzRlClT9Pbbb+vxxx+XdGsnnddee03Dhg3TqFGjTK7QGIMHD9bmzZs1Y8YMtW/fXnv27FHVqlX1xRdf6PXXX9fu3bvNLhFOhrAJSVL58uX16quvauTIkVl2WwEc1V93yYJjsVqtGjlypGbNmqX09HRJkoeHh0aMGKHIyEiTqzNOlSpVtGLFCjVr1sxu56RDhw6pUaNGdtsPA3mBVAFJt8b2dOvWjaAJp7Bw4ULVrVtXHh4e8vDwUN26dbVgwQKzy8IDZrFYNHnyZJ07d04//vijEhISdPHiRYcOmpJ07ty5bBesT01N5RcrmIJkAUm3FgFesWKF2WUAhouMjNTgwYPVsWNHffLJJ/rkk0/UsWNHDR061OFDiLMqXry4GjdurLp168rd3d3scgwXFBSkNWvW2J7fDpgLFiywbdEK5CVuo0PSrUkRS5cuVYMGDVS/fv0sE4SmTZtmUmXAg1W2bFnNmjVL3bt3t2v/+OOPNWjQoGw3NkDBlJqaqkmTJikmJibbndEcdXzupk2b9OSTT+qFF17Q4sWL9Y9//EN79+7Vli1b9P333yswMNDsEuFkmI0OSdJPP/2khg0bSpJ+/vlnu2PcdoEjuXHjhoKCgrK0BwYG6ubNmyZUBKP0799f33//vXr16qUKFSo4zc+yli1bKj4+XpMmTVK9evW0fv16NWrUSHFxcapXr57Z5cEJ0bMJwKkMGjRIhQsXztJbP3z4cF27dk1z5swxqTI8aCVKlNCaNWvUokULs0sBnBo9mwAcXkREhO1ri8WiBQsWaP369WrWrJmkWzvKJCYmKjQ01KwSYYCSJUuqVKlSZpdhioyMDH3++efat2+fJMnf31/PPPMMi7vDFPRsOrEuXbpo8eLF8vLyUpcuXe547meffZZHVQEPXps2be7pPIvFou+++87gapBXPvzwQ33xxRdasmSJihYtanY5eeaXX37R008/raSkJNWsWVPSrYXdy5Ytq//85z+qW7euyRXC2fArjhPz9va2jWHy9vY2uRrAOBs2bDC7BJhg6tSpOnz4sHx8fOTn55dl4uOuXbtMqsxY/fv3V506dbRjxw6VLFlSkvTbb7+pT58+evHFF7VlyxaTK4SzoWcTkqRr164pMzNTxYoVkyQdO3ZMq1evVu3atRUSEmJydQCQe2+88cYdj48bNy6PKslbRYoU0Y4dO1SnTh279p9//lmNGzfWtWvXTKoMzoqeTUiSnnnmGXXp0kUvvfSSLl26pGbNmqlw4cI6f/68pk2bpoEDB5pdIgDkiqOGybt5+OGHlZycnCVsnj17VtWrVzepKjgzFnWHpFu3kx599FFJ0qeffiofHx8dP35cS5cu1axZs0yuDgDuz6VLl7RgwQKNGjVKFy9elHTr592pU6dMrsw4UVFRevXVV/Xpp5/q5MmTOnnypD799FMNGTJEkydP1pUrV2wPIC9wGx2SpKJFi+rXX39V5cqV9dxzz6lOnToaN26cTpw4oZo1a+rq1atmlwgAubJnzx4FBwfL29tbx44d0/79+1W1alWNGTNGiYmJWrp0qdklGuLP2w7fHpd/+5/6Pz+3WCzKyMjI+wLhdLiNDklS9erVtXr1anXu3Flff/21hg4dKunWbRcvLy+TqwOA3IuIiFCfPn00ZcoUeXp62tqfeuop9ejRw8TKjMWEOOQ3hE1IurVfdI8ePTR06FC1bdvWtn/u+vXrbTsLAUBBsn37dr333ntZ2itVqqSkpCQTKsobrVq1uqfzXn75ZdWpU0dlypQxuCI4O8ZsQpL07LPPKjExUTt27NC6dets7W3bttX06dNNrAwA7o+7u3u24xJvrznp7D788EPGbSJPEDZhU758eTVs2NBuvE+TJk1Uq1YtE6sCgPvz9NNPa/z48bpx44akW+MVExMTNWLECHXt2tXk6szHlA3kFcImAMAhTZ06VSkpKSpXrpyuXbumVq1aqXr16vL09NRbb71ldnmA02DMJgDAIXl7e+ubb77R5s2blZCQoJSUFDVq1EjBwcFmlwY4FZY+AgA4pKVLl6pbt25yd3e3a09PT9fy5csVGhpqUmX5g6enpxISElS1alWzS4GDI2wCABySq6urzpw5o3Llytm1X7hwQeXKlXP6NSYJm8grjNkEADik2wuX/9XJkyfl7e1tQkX5ywsvvMA6ysgTjNkEADiUhg0bymKxyGKxqG3btipU6I9/6jIyMnT06FG1b9/exAqNtW3bNsXFxdnWEi1fvryaN2+uJk2a2J03d+5cM8qDEyJsAgAcSqdOnSRJ8fHxCgkJUfHixW3H3Nzc5Ofn55BLH509e1Zdu3bV5s2bVblyZfn4+EiSkpOTNXToULVo0UKrVq3KMqwAMBpjNgEADmnJkiXq1q2bPDw8zC4lTzz77LM6ffq03n//fdWsWdPu2P79+9W3b19VrFhRn3zyiUkVwlkRNgEADi09PV1nz55VZmamXXvlypVNqsgYnp6e+uGHH3LcYnjnzp1q3bq1fv/99zyuDM6O2+gAAId08OBB9e3bV1u2bLFrvz1xyNFmo+e0Pedtv//+e5ZloIC8QNgEADikPn36qFChQvrqq69UoUKFbGemO5Ju3bqpd+/emj59utq2bWubaX7lyhXFxMQoIiJC3bt3N7lKOCNuowMAHFKxYsW0c+dO1apVy+xS8kRaWpqGDBmiRYsW6ebNm3Jzc5N0axhBoUKF1K9fP02fPp3eTeQ5wiYAwCE1btxY06dPV8uWLc0uJU9duXJFO3futFv6KDAwkDU1YRrCJgDAIX333XcaM2aMJk6cqHr16qlw4cJ2xwlfQN4gbAIAHJKLy61N8v46VtNRJwjdTXJyst577z1FRkaaXQqcDGETAOCQvv/++zseb9WqVR5Vkj8kJCSoUaNGTheyYT5mowMAHJKzhck9e/bc8fj+/fvzqBLAHj2bAACHtXHjRr333ns6cuSIPvnkE1WqVEkffPCBHnroIYebOOTi4iKLxaLs/lm/3e6MwwdgPhezCwAAwAirVq1SSEiIihQpol27diktLU2SdPnyZU2cONHk6h68UqVKaf78+Tp69GiWx5EjR/TVV1+ZXSKcFLfRAQAOacKECYqOjlZoaKiWL19ua2/RooUmTJhgYmXGCAwM1OnTp1WlSpVsj1+6dCnbXk/AaIRNAIBD2r9/vx577LEs7d7e3rp06VLeF2Swl156SampqTker1y5st5///08rAi4hbAJAHBI5cuX16FDh+Tn52fXvmnTJlWtWtWcogzUuXPnOx4vWbKkevfunUfVAH9gzCYAwCENGDBAgwcP1tatW2WxWHT69Gl99NFHGj58uAYOHGh2eabz8vLSkSNHzC4DToCeTQCAQxo5cqQyMzPVtm1bXb16VY899pjc3d01fPhwDRo0yOzyTMf4TeQVlj4CADi09PR0HTp0SCkpKfL391fx4sXNLilf8PT0VEJCgkMOKUD+wm10AIBDc3Nzk7+/v2rVqqVvv/1W+/btM7skwKkQNgEADum5557T7NmzJUnXrl1T48aN9dxzz6l+/fpatWqVydUBzoOwCQBwSD/88IMeffRRSdLnn3+uzMxMXbp0SbNmzXLIdTZzy2KxmF0CnARhEwDgkC5fvqxSpUpJktatW6euXbuqaNGi6tChgw4ePGhydeZjygbyCmETAOCQfH19FRcXp9TUVK1bt07t2rWTJP3222/y8PAwubq8YbVacwyV//3vf1WpUqU8rgjOiLAJAHBIQ4YMUc+ePfU///M/qlixolq3bi3p1u31evXqmVucwRYuXKi6devKw8NDHh4eqlu3rhYsWGB3TsuWLeXu7m5ShXAmLH0EAHBYO3fuVGJiop544gnbkkdr1qxRiRIl1KJFC5OrM0ZkZKSmTZumQYMGqXnz5pKkuLg4zZ49W0OHDtX48eNNrhDOhrAJAHBqXl5eio+Pd5j1JsuWLatZs2ape/fudu0ff/yxBg0apPPnz5tUGZwVt9EBAE7N0fpcbty4oaCgoCztgYGBunnzpgkVwdkRNgEAcCC9evXS3Llzs7TPmzdPPXv2NKEiODv2RgcAoICLiIiwfW2xWLRgwQKtX79ezZo1kyRt3bpViYmJCg0NNatEODHCJgAABdzu3bvtngcGBkqSDh8+LEkqU6aMypQpo19++SXPawMImwAAp+YIO+ls2LDB7BKAHDFmEwDg1BxtghCQ39CzCQBweLcDZXa9mI62k06bNm3u2Fv73Xff5WE1AD2bAAAH5ow76QQEBKhBgwa2h7+/v9LT07Vr1y6H3zkJ+RM9mwAAh5TTTjpDhw5VYmKiw+6kM3369GzbX3/9daWkpORxNQA7CAEAHBQ76dg7dOiQmjRpoosXL5pdCpwMt9EBAA6JnXTsxcXFycPDw+wy4IS4jQ4AcEi3d9KZNm2aXbuj76TTpUsXu+dWq1VnzpzRjh07NHbsWJOqgjMjbAIAHAY76Uje3t52z11cXFSzZk2NHz9e7dq1M6kqODPGbAIAHEabNm3u6TyLxcISQEAeIWwCAOCA0tPTdfbsWWVmZtq1V65c2aSK4Ky4jQ4AgAM5cOCA+vXrpy1btti1W61WWSwWZWRkmFQZnBVhEwDgkJx1J52wsDAVKlRIX331lSpUqOAQe7+jYCNsAgAcUkBAgN3zGzduKD4+Xj///LN69+5tTlF5ID4+Xjt37lStWrXMLgWQRNgEADgoZ91Jx9/f3+kWrEf+xgQhAIBTccSddK5cuWL7eseOHRozZowmTpyoevXqqXDhwnbnenl55XV5cHL0bAIAnIoj7qRTokQJu7GZVqtVbdu2tTuHCUIwC2ETAOCQnGknnQ0bNphdApAjbqMDABxSWFiY3XMXFxeVLVtWjz/+ODvpSHr55Zc1fvx4lSlTxuxS4OAImwAAOCEvLy/Fx8eratWqZpcCB8dtdACAQ2MnnezR14S8QtgEADgkdtIB8gfCJgDAIbGTDpA/EDYBAA6JnXSA/MHF7AIAADACO+kA+QNhEwDgMK5cuWJ7TJ48Wa+99ppiY2N14cIFu2N/3nHHEXTp0sV2TUuXLlVaWtpdX/PCCy+wmxDyBEsfAQAchouLS5addP46VtMRJwi5ubnp+PHjqlChglxdXXXmzBmVK1fO7LIASYzZBAA4EGfdSadWrVoaNWqU2rRpI6vVqpUrV+bYaxkaGprH1cHZ0bMJAHBqjrCTzpYtWxQREaHDhw/r4sWL8vT0zHb2vcVi0cWLF02oEM6MsAkAcGqOtpOOi4uLTp06pQoVKti1W61WJSYmqkqVKiZVBmfFBCEAgFNzlj6XixcvOkygRsFC2AQAwMG4urpmaUtJSZGHh4cJ1cDZMUEIAAAHEBERIenWuMzIyEgVLVrUdiwjI0Nbt25VQECASdXBmRE2AQBwALt375Z0a1jATz/9JDc3N9sxNzc3NWjQQMOHDzerPDgxwiYAAA7g9rJPYWFhmjlzJgu2I99gzCYAwGGwk470/vvvO9T1oOBj6SMAgMNgJx0g/+E2OgDAYbCTDpD/0LMJAHAY7KQD5D+ETQCAQ2InHSB/YIIQAMCpsJMOkLcImwAAh8VOOoD5mCAEAHAo7KQD5C+ETQCAQ2EnHSB/YYIQAMAhsZMOkD8QNgEAAGAYJggBAADAMIRNAAAAGIawCQAAAMMQNgEAAGAYwiYAAAAMQ9gEAACAYQibAAAAMMz/B0aNw2a1f+pbAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "all_model_results.sort_values('f1', ascending = False)['f1'].plot(kind = 'bar', figsize = (10, 7))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 780
        },
        "id": "LQ-fLmFmhoMz",
        "outputId": "5e25dc26-102f-4dce-adbd-bb81dc6b96d6"
      },
      "execution_count": 97,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Axes: >"
            ]
          },
          "metadata": {},
          "execution_count": 97
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x700 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzoAAALqCAYAAAD9+CEvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABcrElEQVR4nO3deVyVZeL+8euAAqKAOy6D4pZKqCiokaWZpE7+WswmJi2MkqYFM8lSJ8MyE20hspxwHbVltL0mHcciLRfKBMVWdwU1EDMhUUHh/P7w66kzoIlxzmP3+bxfr/MauZ/nwIVnEq5zP8992+x2u10AAAAAYBAvqwMAAAAAQE2j6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4tawOcD4qKip04MABBQQEyGazWR0HAAAAgEXsdrt+/vlntWjRQl5eZ5+3+UMUnQMHDigkJMTqGAAAAAAuEnl5efrTn/501uN/iKITEBAg6fQ3ExgYaHEaAAAAAFYpLi5WSEiIoyOczR+i6Jy5XC0wMJCiAwAAAOA3b2lhMQIAAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADj1LI6wMUidMIyqyP8LnumD7E6wu/GawAAAICawowOAAAAAONQdAAAAAAY54KKzqxZsxQaGio/Pz/17t1bGzZsOOf5aWlp6tixo+rUqaOQkBCNHTtWJ06cuKDAAAAAAPBbql10li5dqqSkJE2ePFnZ2dnq1q2bBg0apIMHD1Z5/uuvv64JEyZo8uTJ+u677zR//nwtXbpUf//73393eAAAAACoSrWLTmpqqhISEhQfH6+wsDClp6fL399fCxYsqPL89evXq0+fPho+fLhCQ0M1cOBA3Xrrrb85CwQAAAAAF6paRaesrExZWVmKiYn55RN4eSkmJkaZmZlVPufyyy9XVlaWo9js2rVLy5cv17XXXnvWr1NaWqri4mKnBwAAAACcr2otL33o0CGVl5crODjYaTw4OFjff/99lc8ZPny4Dh06pCuuuEJ2u12nTp3SPffcc85L11JSUvTEE09UJxoAAAAAOLh81bXVq1dr2rRp+sc//qHs7Gy98847WrZsmZ588smzPmfixIkqKipyPPLy8lwdEwAAAIBBqjWj07hxY3l7e6ugoMBpvKCgQM2aNavyOY899phuv/12jRo1SpLUpUsXlZSU6O6779ajjz4qL6/KXcvX11e+vr7ViQYAAAAADtWa0fHx8VFkZKQyMjIcYxUVFcrIyFB0dHSVzzl27FilMuPt7S1Jstvt1c0LAAAAAL+pWjM6kpSUlKSRI0cqKipKvXr1UlpamkpKShQfHy9JiouLU8uWLZWSkiJJuu6665Samqru3burd+/e2rFjhx577DFdd911jsIDAAAAADWp2kUnNjZWhYWFSk5OVn5+viIiIrRixQrHAgW5ublOMziTJk2SzWbTpEmTtH//fjVp0kTXXXednnrqqZr7LgAAAADgV2z2P8D1Y8XFxQoKClJRUZECAwNd8jVCJyxzyed1lz3Th1gd4XfjNQAAAMBvOd9u4PJV1wAAAADA3Sg6AAAAAIxT7Xt0AJiLywcBAIApmNEBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4LEYAABcRFoQAAKBmMKMDAAAAwDjM6AAA8H/+6DNqErNqAHAGMzoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHxQgAAMBFgwUhANQUZnQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAOixEAAADAgQUhYApmdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxWHUNAAAAuIiw8l3NYEYHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMc0FFZ9asWQoNDZWfn5969+6tDRs2nPXcq666SjabrdJjyJAhFxwaAAAAAM6l2kVn6dKlSkpK0uTJk5Wdna1u3bpp0KBBOnjwYJXnv/POO/rhhx8cj6+//lre3t76y1/+8rvDAwAAAEBVql10UlNTlZCQoPj4eIWFhSk9PV3+/v5asGBBlec3bNhQzZo1czw++ugj+fv7U3QAAAAAuEy1ik5ZWZmysrIUExPzyyfw8lJMTIwyMzPP63PMnz9ff/3rX1W3bt2znlNaWqri4mKnBwAAAACcr2oVnUOHDqm8vFzBwcFO48HBwcrPz//N52/YsEFff/21Ro0adc7zUlJSFBQU5HiEhIRUJyYAAAAAD+fWVdfmz5+vLl26qFevXuc8b+LEiSoqKnI88vLy3JQQAAAAgAlqVefkxo0by9vbWwUFBU7jBQUFatas2TmfW1JSoiVLlmjKlCm/+XV8fX3l6+tbnWgAAAAA4FCtGR0fHx9FRkYqIyPDMVZRUaGMjAxFR0ef87lvvvmmSktLddttt11YUgAAAAA4T9Wa0ZGkpKQkjRw5UlFRUerVq5fS0tJUUlKi+Ph4SVJcXJxatmyplJQUp+fNnz9fN954oxo1alQzyQEAAADgLKpddGJjY1VYWKjk5GTl5+crIiJCK1ascCxQkJubKy8v54mirVu3au3atVq5cmXNpAYAAACAc6h20ZGkxMREJSYmVnls9erVlcY6duwou91+IV8KAAAAAKrNrauuAQAAAIA7UHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADDOBRWdWbNmKTQ0VH5+furdu7c2bNhwzvOPHDmi+++/X82bN5evr68uueQSLV++/IICAwAAAMBvqVXdJyxdulRJSUlKT09X7969lZaWpkGDBmnr1q1q2rRppfPLysp0zTXXqGnTpnrrrbfUsmVL7d27V/Xr16+J/AAAAABQSbWLTmpqqhISEhQfHy9JSk9P17Jly7RgwQJNmDCh0vkLFizQ4cOHtX79etWuXVuSFBoa+vtSAwAAAMA5VOvStbKyMmVlZSkmJuaXT+DlpZiYGGVmZlb5nA8++EDR0dG6//77FRwcrPDwcE2bNk3l5eVn/TqlpaUqLi52egAAAADA+apW0Tl06JDKy8sVHBzsNB4cHKz8/Pwqn7Nr1y699dZbKi8v1/Lly/XYY4/pueee09SpU8/6dVJSUhQUFOR4hISEVCcmAAAAAA/n8lXXKioq1LRpU82ZM0eRkZGKjY3Vo48+qvT09LM+Z+LEiSoqKnI88vLyXB0TAAAAgEGqdY9O48aN5e3trYKCAqfxgoICNWvWrMrnNG/eXLVr15a3t7djrHPnzsrPz1dZWZl8fHwqPcfX11e+vr7ViQYAAAAADtWa0fHx8VFkZKQyMjIcYxUVFcrIyFB0dHSVz+nTp4927NihiooKx9i2bdvUvHnzKksOAAAAAPxe1b50LSkpSXPnztWiRYv03Xff6d5771VJSYljFba4uDhNnDjRcf69996rw4cPa8yYMdq2bZuWLVumadOm6f7776+57wIAAAAAfqXay0vHxsaqsLBQycnJys/PV0REhFasWOFYoCA3N1deXr/0p5CQEP33v//V2LFj1bVrV7Vs2VJjxozR+PHja+67AAAAAIBfqXbRkaTExEQlJiZWeWz16tWVxqKjo/X5559fyJcCAAAAgGpz+aprAAAAAOBuFB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcS6o6MyaNUuhoaHy8/NT7969tWHDhrOeu3DhQtlsNqeHn5/fBQcGAAAAgN9S7aKzdOlSJSUlafLkycrOzla3bt00aNAgHTx48KzPCQwM1A8//OB47N2793eFBgAAAIBzqXbRSU1NVUJCguLj4xUWFqb09HT5+/trwYIFZ32OzWZTs2bNHI/g4ODfFRoAAAAAzqVaRaesrExZWVmKiYn55RN4eSkmJkaZmZlnfd7Ro0fVunVrhYSE6IYbbtA333xzzq9TWlqq4uJipwcAAAAAnK9qFZ1Dhw6pvLy80oxMcHCw8vPzq3xOx44dtWDBAr3//vt69dVXVVFRocsvv1z79u0769dJSUlRUFCQ4xESElKdmAAAAAA8nMtXXYuOjlZcXJwiIiLUr18/vfPOO2rSpIlmz5591udMnDhRRUVFjkdeXp6rYwIAAAAwSK3qnNy4cWN5e3uroKDAabygoEDNmjU7r89Ru3Ztde/eXTt27DjrOb6+vvL19a1ONAAAAABwqNaMjo+PjyIjI5WRkeEYq6ioUEZGhqKjo8/rc5SXl+urr75S8+bNq5cUAAAAAM5TtWZ0JCkpKUkjR45UVFSUevXqpbS0NJWUlCg+Pl6SFBcXp5YtWyolJUWSNGXKFF122WVq3769jhw5omeeeUZ79+7VqFGjavY7AQAAAID/U+2iExsbq8LCQiUnJys/P18RERFasWKFY4GC3NxceXn9MlH0008/KSEhQfn5+WrQoIEiIyO1fv16hYWF1dx3AQAAAAC/Uu2iI0mJiYlKTEys8tjq1audPn7++ef1/PPPX8iXAQAAAIAL4vJV1wAAAADA3Sg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONcUNGZNWuWQkND5efnp969e2vDhg3n9bwlS5bIZrPpxhtvvJAvCwAAAADnpdpFZ+nSpUpKStLkyZOVnZ2tbt26adCgQTp48OA5n7dnzx6NGzdOV1555QWHBQAAAIDzUe2ik5qaqoSEBMXHxyssLEzp6eny9/fXggULzvqc8vJyjRgxQk888YTatm37uwIDAAAAwG+pVtEpKytTVlaWYmJifvkEXl6KiYlRZmbmWZ83ZcoUNW3aVHfdddd5fZ3S0lIVFxc7PQAAAADgfFWr6Bw6dEjl5eUKDg52Gg8ODlZ+fn6Vz1m7dq3mz5+vuXPnnvfXSUlJUVBQkOMREhJSnZgAAAAAPJxLV137+eefdfvtt2vu3Llq3LjxeT9v4sSJKioqcjzy8vJcmBIAAACAaWpV5+TGjRvL29tbBQUFTuMFBQVq1qxZpfN37typPXv26LrrrnOMVVRUnP7CtWpp69atateuXaXn+fr6ytfXtzrRAAAAAMChWjM6Pj4+ioyMVEZGhmOsoqJCGRkZio6OrnR+p06d9NVXX2nz5s2Ox/XXX6/+/ftr8+bNXJIGAAAAwCWqNaMjSUlJSRo5cqSioqLUq1cvpaWlqaSkRPHx8ZKkuLg4tWzZUikpKfLz81N4eLjT8+vXry9JlcYBAAAAoKZUu+jExsaqsLBQycnJys/PV0REhFasWOFYoCA3N1deXi699QcAAAAAzqnaRUeSEhMTlZiYWOWx1atXn/O5CxcuvJAvCQAAAADnjakXAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYJwLKjqzZs1SaGio/Pz81Lt3b23YsOGs577zzjuKiopS/fr1VbduXUVEROiVV1654MAAAAAA8FuqXXSWLl2qpKQkTZ48WdnZ2erWrZsGDRqkgwcPVnl+w4YN9eijjyozM1NbtmxRfHy84uPj9d///vd3hwcAAACAqlS76KSmpiohIUHx8fEKCwtTenq6/P39tWDBgirPv+qqqzR06FB17txZ7dq105gxY9S1a1etXbv2d4cHAAAAgKpUq+iUlZUpKytLMTExv3wCLy/FxMQoMzPzN59vt9uVkZGhrVu3qm/fvmc9r7S0VMXFxU4PAAAAADhf1So6hw4dUnl5uYKDg53Gg4ODlZ+ff9bnFRUVqV69evLx8dGQIUP04osv6pprrjnr+SkpKQoKCnI8QkJCqhMTAAAAgIdzy6prAQEB2rx5s7788ks99dRTSkpK0urVq896/sSJE1VUVOR45OXluSMmAAAAAEPUqs7JjRs3lre3twoKCpzGCwoK1KxZs7M+z8vLS+3bt5ckRURE6LvvvlNKSoquuuqqKs/39fWVr69vdaIBAAAAgEO1ZnR8fHwUGRmpjIwMx1hFRYUyMjIUHR193p+noqJCpaWl1fnSAAAAAHDeqjWjI0lJSUkaOXKkoqKi1KtXL6WlpamkpETx8fGSpLi4OLVs2VIpKSmSTt9vExUVpXbt2qm0tFTLly/XK6+8opdffrlmvxMAAAAA+D/VLjqxsbEqLCxUcnKy8vPzFRERoRUrVjgWKMjNzZWX1y8TRSUlJbrvvvu0b98+1alTR506ddKrr76q2NjYmvsuAAAAAOBXql10JCkxMVGJiYlVHvvfRQamTp2qqVOnXsiXAQAAAIAL4pZV1wAAAADAnSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONcUNGZNWuWQkND5efnp969e2vDhg1nPXfu3Lm68sor1aBBAzVo0EAxMTHnPB8AAAAAfq9qF52lS5cqKSlJkydPVnZ2trp166ZBgwbp4MGDVZ6/evVq3XrrrVq1apUyMzMVEhKigQMHav/+/b87PAAAAABUpdpFJzU1VQkJCYqPj1dYWJjS09Pl7++vBQsWVHn+a6+9pvvuu08RERHq1KmT5s2bp4qKCmVkZPzu8AAAAABQlWoVnbKyMmVlZSkmJuaXT+DlpZiYGGVmZp7X5zh27JhOnjyphg0bnvWc0tJSFRcXOz0AAAAA4HxVq+gcOnRI5eXlCg4OdhoPDg5Wfn7+eX2O8ePHq0WLFk5l6X+lpKQoKCjI8QgJCalOTAAAAAAezq2rrk2fPl1LlizRu+++Kz8/v7OeN3HiRBUVFTkeeXl5bkwJAAAA4I+uVnVObty4sby9vVVQUOA0XlBQoGbNmp3zuc8++6ymT5+ujz/+WF27dj3nub6+vvL19a1ONAAAAABwqNaMjo+PjyIjI50WEjizsEB0dPRZn/f000/rySef1IoVKxQVFXXhaQEAAADgPFRrRkeSkpKSNHLkSEVFRalXr15KS0tTSUmJ4uPjJUlxcXFq2bKlUlJSJEkzZsxQcnKyXn/9dYWGhjru5alXr57q1atXg98KAAAAAJxW7aITGxurwsJCJScnKz8/XxEREVqxYoVjgYLc3Fx5ef0yUfTyyy+rrKxMN998s9PnmTx5sh5//PHflx4AAAAAqlDtoiNJiYmJSkxMrPLY6tWrnT7es2fPhXwJAAAAALhgbl11DQAAAADcgaIDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMM4FFZ1Zs2YpNDRUfn5+6t27tzZs2HDWc7/55hsNGzZMoaGhstlsSktLu9CsAAAAAHBeql10li5dqqSkJE2ePFnZ2dnq1q2bBg0apIMHD1Z5/rFjx9S2bVtNnz5dzZo1+92BAQAAAOC3VLvopKamKiEhQfHx8QoLC1N6err8/f21YMGCKs/v2bOnnnnmGf31r3+Vr6/v7w4MAAAAAL+lWkWnrKxMWVlZiomJ+eUTeHkpJiZGmZmZNRaqtLRUxcXFTg8AAAAAOF/VKjqHDh1SeXm5goODncaDg4OVn59fY6FSUlIUFBTkeISEhNTY5wYAAABgvoty1bWJEyeqqKjI8cjLy7M6EgAAAIA/kFrVOblx48by9vZWQUGB03hBQUGNLjTg6+vL/TwAAAAALli1ZnR8fHwUGRmpjIwMx1hFRYUyMjIUHR1d4+EAAAAA4EJUa0ZHkpKSkjRy5EhFRUWpV69eSktLU0lJieLj4yVJcXFxatmypVJSUiSdXsDg22+/dfx5//792rx5s+rVq6f27dvX4LcCAAAAAKdVu+jExsaqsLBQycnJys/PV0REhFasWOFYoCA3N1deXr9MFB04cEDdu3d3fPzss8/q2WefVb9+/bR69erf/x0AAAAAwP+odtGRpMTERCUmJlZ57H/LS2hoqOx2+4V8GQAAAAC4IBflqmsAAAAA8HtQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGuaCiM2vWLIWGhsrPz0+9e/fWhg0bznn+m2++qU6dOsnPz09dunTR8uXLLygsAAAAAJyPahedpUuXKikpSZMnT1Z2dra6deumQYMG6eDBg1Wev379et1666266667tGnTJt1444268cYb9fXXX//u8AAAAABQlWoXndTUVCUkJCg+Pl5hYWFKT0+Xv7+/FixYUOX5L7zwggYPHqyHH35YnTt31pNPPqkePXropZde+t3hAQAAAKAq1So6ZWVlysrKUkxMzC+fwMtLMTExyszMrPI5mZmZTudL0qBBg856PgAAAAD8XrWqc/KhQ4dUXl6u4OBgp/Hg4GB9//33VT4nPz+/yvPz8/PP+nVKS0tVWlrq+LioqEiSVFxcXJ241VJResxln9sdXPl34y68BtbjNbAer4G1/uh//xKvwcWA18B6vAbWc+VrcOZz2+32c55XraLjLikpKXriiScqjYeEhFiQ5o8hKM3qBOA1sB6vgfV4DazHa2A9XgPr8RpYzx2vwc8//6ygoKCzHq9W0WncuLG8vb1VUFDgNF5QUKBmzZpV+ZxmzZpV63xJmjhxopKSkhwfV1RU6PDhw2rUqJFsNlt1Il8UiouLFRISory8PAUGBlodxyPxGliP18B6vAbW4zWwHq+Btfj7t54Jr4HdbtfPP/+sFi1anPO8ahUdHx8fRUZGKiMjQzfeeKOk0yUkIyNDiYmJVT4nOjpaGRkZevDBBx1jH330kaKjo8/6dXx9feXr6+s0Vr9+/epEvSgFBgb+Yf8PZQpeA+vxGliP18B6vAbW4zWwFn//1vujvwbnmsk5o9qXriUlJWnkyJGKiopSr169lJaWppKSEsXHx0uS4uLi1LJlS6WkpEiSxowZo379+um5557TkCFDtGTJEm3cuFFz5syp7pcGAAAAgPNS7aITGxurwsJCJScnKz8/XxEREVqxYoVjwYHc3Fx5ef2ymNvll1+u119/XZMmTdLf//53dejQQe+9957Cw8Nr7rsAAAAAgF+5oMUIEhMTz3qp2urVqyuN/eUvf9Ff/vKXC/lSRvD19dXkyZMrXY4H9+E1sB6vgfV4DazHa2A9XgNr8fdvPU96DWz231qXDQAAAAD+YKq1YSgAAAAA/BFQdAAAAAAYh6IDAAAAwDgUHQAAAADGoejAOCdPntSdd96p3bt3Wx0FAAAAFmHVNRgpKChImzdvVps2bayOAlhuzZo1mj17tnbu3Km33npLLVu21CuvvKI2bdroiiuusDqexzhx4oTKysqcxv7Iu5L/UeTm5p7zeKtWrdyUBIC7XdA+OqhaUlLSeZ+bmprqwiS48cYb9d5772ns2LFWR/F4p06d0urVq7Vz504NHz5cAQEBOnDggAIDA1WvXj2r4xnv7bff1u23364RI0Zo06ZNKi0tlSQVFRVp2rRpWr58ucUJzXbs2DE98sgjeuONN/Tjjz9WOl5eXm5BKs8SGhoqm8121uO8BoC5KDo1aNOmTU4fZ2dn69SpU+rYsaMkadu2bfL29lZkZKQV8TxKhw4dNGXKFK1bt06RkZGqW7eu0/EHHnjAomSeZe/evRo8eLByc3NVWlqqa665RgEBAZoxY4ZKS0uVnp5udUTjTZ06Venp6YqLi9OSJUsc43369NHUqVMtTOYZHn74Ya1atUovv/yybr/9ds2aNUv79+/X7NmzNX36dKvjeYT//dl88uRJbdq0SampqXrqqacsSuV5Tpw4oRdffFGrVq3SwYMHVVFR4XQ8OzvbomRma9CgwTmL/q8dPnzYxWncj6JTg1atWuX4c2pqqgICArRo0SI1aNBAkvTTTz8pPj5eV155pVURPcb8+fNVv359ZWVlKSsry+mYzWaj6LjJmDFjFBUVpZycHDVq1MgxPnToUCUkJFiYzHNs3bpVffv2rTQeFBSkI0eOuD+Qh/n3v/+txYsX66qrrnL8+9++fXu1bt1ar732mkaMGGF1RON169at0lhUVJRatGihZ555RjfddJMFqTzPXXfdpZUrV+rmm29Wr169zvuXb/w+aWlpVkewFEXHRZ577jmtXLnSUXKk06166tSpGjhwoB566CEL05mPhQguDmvWrNH69evl4+PjNB4aGqr9+/dblMqzNGvWTDt27FBoaKjT+Nq1a9W2bVtrQnmQw4cPO/6eAwMDHe+YXnHFFbr33nutjObxOnbsqC+//NLqGB7jww8/1PLly9WnTx+ro3iUkSNHWh3BUqy65iLFxcUqLCysNF5YWKiff/7ZgkSeqaysTFu3btWpU6esjuKRKioqqrz+fd++fQoICLAgkedJSEjQmDFj9MUXX8hms+nAgQN67bXXNG7cOH7RdoO2bds63njp1KmT3njjDUmnZ3rq169vYTLPUVxc7PQoKirS999/r0mTJqlDhw5Wx/MYLVu25N/9i8iJEycq/bdhJDtc4vbbb7eHhoba3377bXteXp49Ly/P/tZbb9nbtGljj4uLszqe8UpKSux33nmn3dvb2+7t7W3fuXOn3W632xMTE+0pKSkWp/Mct9xyiz0hIcFut9vt9erVs+/atcv+888/26+++mr7HXfcYXE6z1BRUWGfOnWqvW7dunabzWa32Wx2Pz8/+6RJk6yO5hFSU1PtL7zwgt1ut9s/+ugju5+fn93X19fu5eVlT0tLszidZ7DZbHYvLy+nh81ms7dq1cq+fv16q+N5jOXLl9sHDx5s37Nnj9VRPNbRo0ft999/v71JkyaV/pvw8vKyOp5LsLy0ixw7dkzjxo3TggULdPLkSUlSrVq1dNddd+mZZ56pdHM8ataYMWO0bt06paWlafDgwdqyZYvatm2r999/X48//nilm1PhGvv27dOgQYNkt9u1fft2RUVFafv27WrcuLE+++wzNW3a1OqIHqOsrEw7duzQ0aNHFRYWxop3Ftm7d6+ysrLUvn17de3a1eo4HuHTTz91+tjLy0tNmjRR+/btVasWV/C7S2FhoW655RZ99tln8vf3V+3atZ2Om3gj/MXm/vvv16pVq/Tkk09WuTiKifcMUnRcrKSkRDt37pQktWvXjoLjJq1bt9bSpUt12WWXKSAgQDk5OWrbtq127NihHj16mDtFexE6deqUlixZoi1btujo0aPq0aOHRowYoTp16lgdzSMVFxfrk08+UceOHdW5c2er4xhv8eLFio2Nla+vr9N4WVmZlixZori4OIuSeYaTJ0/qb3/7mx577DH2VbNYTEyMcnNzdddddyk4OLjSYgSefi+JO7Rq1cqxOEpgYKCys7PVvn17vfLKK/rXv/5l5HYDFB0Yyd/fX19//bXatm3rVHRycnLUt29fFRUVWR0RcItbbrlFffv2VWJioo4fP66IiAjt3r1bdrtdS5Ys0bBhw6yOaDRvb2/98MMPlWYvf/zxRzVt2pQ9XNyADaQvDv7+/srMzKxyFTy4R7169fTtt9+qVatW+tOf/qR33nlHvXr10u7du9WlSxcdPXrU6og1jsUI3Gznzp26+uqrrY5hvKioKC1btszx8Zl3jubNm6fo6GirYnmk7du3a86cOZo6daqmTJni9IDrffbZZ44l7d99911VVFToyJEjmjlzJvvouIHdbq9yGd19+/YpKCjIgkSe58wG0rBWp06ddPz4catjeDRPXByFi1Pd7OjRo5WuF0bNmzZtmv785z/r22+/1alTp/TCCy/o22+/1fr16/n7d6O5c+fq3nvvVePGjdWsWTOnX/hsNpuSk5MtTOcZioqK1LBhQ0nSihUrNGzYMPn7+2vIkCF6+OGHLU5nru7du8tms8lms2nAgAFO94KUl5dr9+7dGjx4sIUJPQcbSF8cpk+froceekhPPfWUunTpUukencDAQIuSeY74+Hjl5OSoX79+mjBhgq677jq99NJLOnnypFJTU62O5xJculbDZs6cec7j+/fv17PPPsvlCm6wc+dOTZ8+XTk5OY57Q8aPH68uXbpYHc1jtG7dWvfdd5/Gjx9vdRSPdckll2jq1KkaMmSI2rRpoyVLlujqq69WTk6OBgwYoEOHDlkd0UhPPPGE438feughp8UffHx8FBoaqmHDhlXaYwo171yXrNlsNu3atcuNaTyXl9fpi4j+d4bzzKwnvxe5nycsjkLRqWFeXl5q3rz5WX94lZWVKT8/n/+g4RECAwO1efNmNqa00D/+8Q+NGTNG9erVU+vWrZWdnS0vLy+9+OKLeuedd7Rq1SqrIxpt0aJFio2NlZ+fn9VRAEv91tUU/fr1c1MSeBKKTg1r06aNZsyYoVtuuaXK45s3b1ZkZCRFxwWqs5IaU+Tucdddd6lnz5665557rI7i0TZu3Ki8vDxdc801jpmFZcuWqX79+uxSDuMlJSVVOW6z2eTn56f27dvrhhtucFziCdfIzc1VSEhIlTM6eXl5atWqlUXJPEtGRoYyMjJ08OBBVVRUOB1bsGCBRalch6JTw26++Wa1a9dOM2bMqPJ4Tk6OunfvXun/XPj9vLy8qrzptyoUTfdISUlRamqqhgwZUuU12VwbD9OVl5fr+eef1xtvvKHc3FyVlZU5HWfvENfr37+/srOzVV5ero4dO0qStm3bJm9vb3Xq1Elbt26VzWbT2rVrFRYWZnFac7ECofWeeOIJTZkyRVFRUWrevHml35neffddi5K5DkWnhn377bc6duyYoqKiqjx+8uRJHThwQK1bt3ZzMvP9elp8z549mjBhgu644w7HKmuZmZlatGiRUlJSWK/fTbg23nrl5eVauHDhWd/B++STTyxK5hmSk5M1b948PfTQQ5o0aZIeffRR7dmzR++9956Sk5Mp+26QlpamNWvW6J///KdjNr+oqEijRo3SFVdcoYSEBA0fPlzHjx/Xf//7X4vTmsvLy0sFBQVq0qSJ0/jevXsVFhamkpISi5J5jubNm+vpp5/W7bffbnUUt6HowEgDBgzQqFGjdOuttzqNv/7665ozZ45Wr15tTTDAzRITE7Vw4UINGTKkynfwnn/+eYuSeYZ27dpp5syZGjJkiAICArR582bH2Oeff67XX3/d6ojGa9mypT766KNKszXffPONBg4cqP379ys7O1sDBw5kcQ4XOHPp4AsvvKCEhAT5+/s7jpWXl+uLL76Qt7e31q1bZ1VEj9GoUSNt2LBB7dq1szqK27C8tItMnTpVI0aMYIMyi2RmZio9Pb3SeFRUlEaNGmVBIsAaS5Ys0RtvvKFrr73W6igeKT8/37HSY7169RybFf+///f/9Nhjj1kZzWMUFRXp4MGDlYpOYWGh497O+vXrV7qsEDVj06ZNkk7fi/PVV185Ldbk4+Ojbt26ady4cVbF8yijRo3S66+/7lH/9lB0XOTNN9/U5MmT1bt3b91222265ZZb1LhxY6tjeYyQkBDNnTtXTz/9tNP4vHnzFBISYlEqz5CUlKQnn3xSdevWPetNwGeYum7/xcTHx0ft27e3OobH+tOf/qQffvhBrVq1Urt27bRy5Ur16NFDX375pXx9fa2O5xFuuOEG3XnnnXruuefUs2dPSdKXX36pcePG6cYbb5QkbdiwQZdccomFKc11ZmXH+Ph4vfDCCywGZKETJ05ozpw5+vjjj9W1a9dK982a+DOZS9dc6JtvvtFrr72mJUuWaN++fbrmmms0YsQI3XjjjU5Tt6h5y5cv17Bhw9S+fXv17t1b0ukfZNu3b9fbb7/Nu9su1L9/f7377ruqX7+++vfvf9bzbDYb94e4wXPPPaddu3bppZdeOu/FOlBzJkyYoMDAQP3973/X0qVLddtttyk0NFS5ubkaO3aspk+fbnVE4x09elRjx47V4sWLderUKUlSrVq1NHLkSD3//POqW7euNm/eLEmKiIiwLqiHKS4u1ieffKJOnTqpU6dOVsfxCJ74M5mi4ybr1q3T66+/rjfffFMnTpyo1lLIuDD79u3TP/7xD33//feSpM6dO+uee+5hRgceZejQoVq1apUaNmyoSy+9tNI7eO+8845FyTxTZmamMjMz1aFDB1133XVWx/EoR48edSyA0rZtW6dNXOF6t9xyi/r27avExEQdP35c3bp10549e2S327VkyRINGzbM6ogwEJeuuUndunVVp04d+fj46Oeff7Y6jkf405/+pGnTplkdA7BU/fr1NXToUKtj4P9ER0c7VoKEe9WrV8/Y3d//CD777DM9+uijkk4vY2y323XkyBEtWrRIU6dOpei42b59+ySd/l3JZMzouNDu3bv1+uuv6/XXX9fWrVvVr18/DR8+XDfffLOCgoKsjme8I0eOaP78+fruu+8kSZdeeqnuvPNO/u5d7Kabbjrvc5lNgIk++OCD8z73+uuvd2ES4OJRp04dbdu2TSEhIYqLi1OLFi00ffp05ebmKiwsTEePHrU6ovEqKio0depUPffcc46/74CAAD300EN69NFH5eXlZXHCmseMjotcdtll+vLLL9W1a1fFx8fr1ltvVcuWLa2O5TE2btyoQYMGqU6dOurVq5ek0zfZPfXUU46bgeEaFMmLz6lTp7R69Wrt3LlTw4cPV0BAgA4cOKDAwEAu33GBMze4n2Gz2fS/7ymeuV+KTRLhKUJCQpSZmamGDRtqxYoVWrJkiSTpp59+kp+fn8XpPMOjjz6q+fPna/r06erTp48kae3atXr88cd14sQJPfXUUxYnrHnM6LjIo48+qhEjRrDLskWuvPJKtW/fXnPnzlWtWqf7/KlTpzRq1Cjt2rVLn332mcUJAffYu3evBg8erNzcXJWWlmrbtm1q27atxowZo9LS0iqXYUfN+fjjjzV+/HhNmzbNafPiSZMmadq0abrmmmssTgi4xz/+8Q+NGTNG9erVU6tWrbRp0yZ5eXnpxRdf1DvvvONYnQ2u06JFC6Wnp1eaSX7//fd13333af/+/RYlcx2KDoxUp04dbdq0qdJKLt9++62ioqJ07Ngxi5J5HmYTrHXjjTcqICBA8+fPV6NGjZSTk6O2bdtq9erVSkhI0Pbt262OaLTw8HClp6friiuucBpfs2aN7r77bseltYAnyMrKUm5urgYOHKi6detKkpYtW6YGDRro8ssvtzid+fz8/LRly5ZKS6lv3bpVEREROn78uEXJXIdL11ykvLxcCxcuVEZGhg4ePKiKigqn4yYu4XcxCQwMVG5ubqWik5eXp4CAAItSeZ7/nU245pprFBAQoBkzZjCb4CZr1qzR+vXrnTbpk6TQ0FAj37272OzcuVP169evNB4UFKQ9e/a4PQ/gTmfbS23NmjWVxig6rtetWze99NJLmjlzptP4Sy+9pG7dulmUyrUoOi4yZswYLVy4UEOGDFF4eDj7V7hZbGys7rrrLj377LOOfzzXrVunhx9+WLfeeqvF6TzHmDFjFBUVpZycHDVq1MgxPnToUCUkJFiYzHNUVFRUeR/Ivn37KP1u0LNnTyUlJemVV15RcHCwJKmgoEAPP/yw4/5BwFSbNm06r/P4Hck9nn76aQ0ZMkQff/yx06W0eXl5Wr58ucXpXINL11ykcePGWrx4MRtTWqSsrEwPP/yw0tPTHRvE1a5dW/fee6+mT5/OjuRu0qhRI61fv14dO3ZUQECA47KpPXv2KCwsjEsI3SA2NlZBQUGaM2eOAgICtGXLFjVp0kQ33HCDWrVqpX/+859WRzTajh07NHToUMdqU9LpmeUOHTrovffeU/v27S1OCMCTHDhwQLNmzXLaY/C+++5TixYtLE7mGhQdF2nRooVWr15d6TpIuNexY8e0c+dOSVK7du3k7+9vcSLP0qBBA61bt05hYWFORWft2rUaNmyYCgoKrI5ovH379mnQoEGy2+3avn27oqKitH37djVu3FifffaZmjZtanVE49ntdn300UdOv1jExMTwLjYAuBhFx0Wee+457dq1Sy+99BI/zCxQVFSk8vJyNWzY0Gn88OHDqlWrlgIDAy1K5lmYTbg4nDp1SkuWLNGWLVt09OhR9ejRQyNGjFCdOnWsjob/06VLFy1fvtwx6wMANWHLli0KDw+Xl5eXtmzZcs5zTdxQl6LjIkOHDtWqVavUsGFDXXrppapdu7bTcTZKdK0///nPuu6663Tfffc5jaenp+uDDz4w9lrUiw2zCcD5+fWMJwDUFC8vL+Xn56tp06by8vKqcl8v6fR9Uibu60XRcZH4+PhzHuedbNdq2LCh1q1bp86dOzuNf//99+rTp49+/PFHi5J5nlOnTmnp0qXKyclhNsFNPvjgg/M+93/3U4A1KDoAXGHv3r1q1aqVbDab9u7de85zW7du7aZU7kPRgZHq1q2rzz//XF26dHEa/+qrr9S7d29ugofRvLy8nD6u6h28M5fUmvgO3h8RRQeAq3322We6/PLLHRupn3Hq1CmtX79effv2tSiZ63j99in4PQoLC7V27VqtXbtWhYWFVsfxGL169dKcOXMqjaenpysyMtKCRJ5p0aJFWrZsmePjRx55RPXr19fll1/+m+8s4cJVVFQ4HitXrlRERIT+85//6MiRIzpy5Ij+85//qEePHlqxYoXVUQEAbtK/f38dPny40nhRUZH69+9vQSLXY0bHRUpKSjR69GgtXrzYsVmot7e34uLi9OKLL7L6l4utW7dOMTEx6tmzpwYMGCBJysjI0JdffqmVK1fqyiuvtDihZ+jYsaNefvllXX311crMzNSAAQOUlpamDz/8ULVq1eJeNTcIDw9Xenq6rrjiCqfxNWvW6O6779Z3331nUTL8GjM6AFzNy8tLBQUFatKkidP4tm3bFBUVpeLiYouSuQ4bhrpIUlKSPv30U/373/9Wnz59JElr167VAw88oIceekgvv/yyxQnN1qdPH2VmZuqZZ57RG2+8oTp16qhr166aP3++OnToYHU8j5GXl+fYJ+S9997TzTffrLvvvlt9+vTRVVddZW04D7Fz507Vr1+/0nhQUJD27Nnj9jwAAPe66aabJJ2+ZPmOO+5w2kuwvLxcW7ZscWyubhqKjou8/fbbeuutt5x+mbv22mtVp04d3XLLLRQdN4iIiNBrr71mdQyPVq9ePf34449q1aqVVq5cqaSkJEmSn5+fjh8/bnE6z9CzZ08lJSXplVdeUXBwsCSpoKBADz/8sHr16mVxOvMtXrxYsbGxlTYpLisr05IlSxQXFydJmj17tuP1AYCaFBQUJOn0nl4BAQFOiwH5+PjosssuU0JCglXxXIpL11zE399fWVlZlVb9+uabb9SrVy+VlJRYlMxzVFRUaMeOHTp48KDj8sEzTLzh7mI0YsQIff/99+revbv+9a9/KTc3V40aNdIHH3ygv//97/r666+tjmi8HTt2aOjQodq2bZtjj5a8vDx16NBB7733nmPGDa7h7e2tH374odJS6j/++KOaNm3KYhAA3OaJJ57Qww8/7FG3T1B0XGTAgAFq1KiRFi9eLD8/P0nS8ePHNXLkSB0+fFgff/yxxQnN9vnnn2v48OHau3dvlatN8cuFexw5ckSTJk1SXl6e7r33Xg0ePFiSNHnyZPn4+OjRRx+1OKFnsNvt+uijj/T9999Lkjp37qyYmBg2M3aDs10Tn5OTc9YbgwHAFXbv3q1Tp05VuoR/+/btql27tkJDQ60J5kIUHRf56quvNHjwYJWWlqpbt26STv9g8/X11cqVK3XppZdanNBsERERuuSSS/TEE0+oefPmlX6hOzONC+C0Ll26aPny5Y5ZH/w+3bt3l81mU05Oji699FKn5VzLy8u1e/duDR48WG+88YaFKQF4kn79+unOO+/UyJEjncZfffVVzZs3T6tXr7YmmAtRdFzo2LFjeu2115zeRWWjRPeoW7eucnJyuCznInHs2DHl5uaqrKzMabxr164WJcL/YtWvmvXEE084/vehhx5SvXr1HMd8fHwUGhqqYcOGycfHx6qIADxMYGCgsrOzK/1utGPHDkVFRenIkSPWBHMhFiNwkZSUFAUHB1e6uWvBggUqLCzU+PHjLUrmGXr37q0dO3ZQdCxWWFioO+6446z7tXAJIUw1efJkSVJoaKhiY2MdlzADgFVsNpt+/vnnSuNFRUXG/jxmw1AXmT17tjp16lRp/NJLL1V6eroFiTzL6NGj9dBDD2nhwoXKysrSli1bnB5wjwcffFBFRUX64osvVKdOHa1YsUKLFi1Shw4d9MEHH1gdD3C5kSNHys/PT2VlZdq3b59yc3OdHgDgLn379lVKSopTqSkvL1dKSkqlvdZMwaVrLuLn56fvvvtObdq0cRrftWuXwsLCdOLECYuSeQYvr8od3mazyW63sxiBGzVv3lzvv/++evXqpcDAQG3cuFGXXHKJPvjgAz399NNau3at1RHxf7h0zTW2b9+uO++8U+vXr3ca598iAO727bffqm/fvqpfv75j4/Q1a9aouLhYn3zyicLDwy1OWPO4dM1FQkJCtG7dukpFZ926dWrRooVFqTzH7t27rY4ASSUlJY5ldRs0aKDCwkJdcskl6tKli7Kzsy1OB7jeHXfcoVq1aunDDz+scmEUAHCXsLAwbdmyRS+99JJycnJUp04dxcXFKTExUQ0bNrQ6nktQdFwkISFBDz74oE6ePKmrr75akpSRkaFHHnlEDz30kMXpzNe6dWurI0BSx44dtXXrVoWGhqpbt26aPXu2QkNDlZ6erubNm1sdD3C5zZs3Kysrq8pLmQHA3Vq0aKFp06ZZHcNtKDou8vDDD+vHH3/Ufffd51hpys/PT+PHj9fEiRMtTucZXnnlFaWnp2v37t3KzMxU69atlZaWpjZt2uiGG26wOp5HGDNmjH744QdJp2/OHjx4sF599VX5+Pho0aJFFqfzPCdOnDjrTfGzZ89WcHCwmxOZLywsTIcOHbI6BgBIOn2p2uzZs7Vr1y69+eabatmypV555RW1adPGyPt0WIzARWw2m2bMmKHCwkJ9/vnnysnJ0eHDh5WcnGx1NI/w8ssvKykpSddee62OHDniuA6+fv36SktLszacB7ntttt0xx13SJJ69OihvXv3auPGjdq3b59iY2OtDechKioq9OSTT6ply5aqV6+edu3aJUl67LHHNH/+fMd5w4cPV926da2KaawZM2bokUce0erVq/Xjjz+quLjY6QEA7vL2229r0KBBqlOnjrKzs1VaWirp9Kprps7yUHRcrF69eurZs6fCw8Pl6+trdRyP8eKLL2ru3Ll69NFH5e3t7RiPiorSV199ZWEyzzN//nyFh4fLz89PDRo0UFxcnN577z2rY3mMqVOnauHChXr66aed9mwJDw/XvHnzLEzmGWJiYvT5559rwIABatq0qRo0aKAGDRqofv36atCggdXxAHiQqVOnKj09XXPnzlXt2rUd43369DH2vlkuXYORdu/ere7du1ca9/X1VUlJiQWJPFNycrJSU1M1evRoRUdHS5IyMzM1duxY5ebmasqUKRYnNN/ixYs1Z84cDRgwQPfcc49jvFu3bo7NjOE6q1atsjoCAEiStm7dqr59+1YaDwoKMnKzUImiA0O1adNGmzdvrrQowYoVK9S5c2eLUnmel19+WXPnztWtt97qGLv++uvVtWtXjR49mqLjBvv3769y49yKigqdPHnSgkSepV+/flZHAABJUrNmzbRjxw6FhoY6ja9du9bYrQW4dA1GSkpK0v3336+lS5fKbrdrw4YNeuqppzRx4kQ98sgjVsfzGCdPnlRUVFSl8cjISJ06dcqCRJ4nLCxMa9asqTT+1ltvVTnriZq3Zs0a3Xbbbbr88su1f/9+SacXS2EfKQDulJCQoDFjxuiLL76QzWbTgQMH9Nprr2ncuHG69957rY7nEszowEijRo1SnTp1NGnSJB07dkzDhw9XixYt9MILL+ivf/2r1fE8xu23366XX35ZqampTuNz5szRiBEjLErlWZKTkzVy5Ejt379fFRUVeuedd7R161YtXrxYH374odXxjPf222/r9ttv14gRI6q8+Xf58uUWJwTgKSZMmKCKigoNGDBAx44dU9++feXr66tx48Zp9OjRVsdzCZvdbrdbHQJwpWPHjuno0aOOjSt/bd26dYqKimKhiBqUlJTk+POpU6e0cOFCtWrVSpdddpkk6YsvvlBubq7i4uL04osvWhXTo6xZs0ZTpkxRTk6Ojh49qh49eig5OVkDBw60OprxunfvrrFjxyouLk4BAQHKyclR27ZttWnTJv35z39Wfn6+1REBeIDy8nKtW7dOXbt2lb+/v3bs2KGjR48qLCxM9erVszqey1B04NECAwO1efNmY69NtUL//v3P6zybzaZPPvnExWkAa/n7++vbb79VaGioU9HZtWuXwsLCdOLECasjAvAQfn5++u6779SmTRuro7gNl67Bo9Hzax6rTAG/8MSbfwFcnMLDw7Vr1y6KDgDgj6tBgway2Wznde7hw4ddnMaznbn5d8GCBY6bfzMzMzVu3Dg99thjVscD4EGmTp2qcePG6cknn1RkZGSlTaIDAwMtSuY6FB0AMExaWprVEfB/PPHmXwAXp2uvvVbS6W0efv1mmN1ul81mU3l5uVXRXIZ7dODRfn3NPAC4SllZmcfc/Avg4vTpp5+e87iJ+35RdODRWIwAnqC8vFzvvvuuvvvuO0mn99a54YYbVKsWk/quVlRUpPLycjVs2NBp/PDhw6pVq5aRl4oAwMWCn3LwaPR8mO6bb77R9ddfr/z8fHXs2FGSNGPGDDVp0kT//ve/FR4ebnFCs/31r3/Vddddp/vuu89p/I033tAHH3zAPjoAXGrLli0KDw+Xl5eXtmzZcs5zu3bt6qZU7sOMDox16tQprV69Wjt37tTw4cMVEBCgAwcOKDAwkMtG4DGio6PVpEkTLVq0SA0aNJAk/fTTT7rjjjtUWFio9evXW5zQbA0bNtS6devUuXNnp/Hvv/9effr00Y8//mhRMgCewMvLS/n5+WratKm8vLxks9mqfJPX1Ht0mNGBkfbu3avBgwcrNzdXpaWluuaaaxQQEKAZM2aotLRU6enpVkcE3GLz5s3auHGjo+RIp1dle+qpp9SzZ08Lk3mG0tJSnTp1qtL4yZMndfz4cQsSAfAku3fvVpMmTRx/9jReVgcAXGHMmDGKiorSTz/9pDp16jjGhw4dqoyMDAuTAe51ySWXqKCgoNL4wYMH1b59ewsSeZZevXppzpw5lcbT09MVGRlpQSIAnqR169aOFdZat259zoeJmNGBkdasWaP169fLx8fHaTw0NFT79++3KBXgfikpKXrggQf0+OOP67LLLpMkff7555oyZYpmzJih4uJix7ncGF/zpk6dqpiYGOXk5GjAgAGSpIyMDH355ZdauXKlxekAmO6DDz4473Ovv/56FyaxBvfowEgNGjTQunXrFBYW5rSE9Nq1azVs2LAq3+EGTOTl9cvE/Zl39c78s//rj029PvtikJOTo6efflqbN29WnTp11LVrV02cOFEdOnSwOhoAw/36Z4CkSvfo/Ho/HRN/BjCjAyMNHDhQaWlpjktGbDabjh49qsmTJzs2zAI8wapVq6yO4LFOnjypv/3tb3rsscf02muvWR0HgAeqqKhw/Pnjjz/W+PHjNW3aNEVHR0uSMjMzNWnSJE2bNs2qiC7FjA6MtG/fPg0aNEh2u13bt29XVFSUtm/frsaNG+uzzz5T06ZNrY4IwAMEBQVp8+bNatOmjdVRAHi48PBwpaen64orrnAaX7Nmje6++27HXmsmoejAWKdOndLSpUuVk5Ojo0ePqkePHhoxYoTT4gSAJzhx4oS2bNmigwcPOr27J5l5TfbFZOTIkYqIiNDYsWOtjgLAw9WpU0dffvllpf3TtmzZot69exu5EiRFBwAMtmLFCsXFxenQoUOVjnFfjutNnTpVzz33nAYMGKDIyEjVrVvX6fgDDzxgUTIAnqZv377y8/PTK6+8ouDgYElSQUGB4uLidOLECX366acWJ6x5FB0YKSUlRcHBwbrzzjudxhcsWKDCwkKNHz/eomSAe3Xo0EEDBw5UcnKy4wcb3Odcl6zZbDbt2rXLjWkAeLIdO3Zo6NCh2rZtm0JCQiRJeXl56tChg9577z0jtxyg6MBIoaGhev3113X55Zc7jX/xxRf661//6pGbZsEzBQYGatOmTWrXrp3VUQAAFrPb7froo4/0/fffS5I6d+6smJgYp9XXTMKqazBSfn6+mjdvXmm8SZMm+uGHHyxIBFjj5ptv1urVqyk6FisrK9Pu3bvVrl071arFj14A1rDZbBo4cKAGDhxodRS34F9bGCkkJETr1q2rdNnIunXr1KJFC4tSAe730ksv6S9/+YvWrFmjLl26qHbt2k7HuUfEtY4dO6bRo0dr0aJFkqRt27apbdu2Gj16tFq2bKkJEyZYnBCAyWbOnKm7775bfn5+mjlz5jnPNfHnAZeuwUhPP/20nn76aT3zzDO6+uqrJZ3ejfyRRx7RQw89pIkTJ1qcEHCP+fPn65577pGfn58aNWrkdHkC94i43pgxY7Ru3TqlpaVp8ODB2rJli9q2bav3339fjz/+uDZt2mR1RAAGa9OmjTZu3KhGjRp55D2DFB0YyW63a8KECZo5c6bKysokSX5+fho/frySk5MtTge4T7NmzfTAAw9owoQJlXbIhuu1bt1aS5cu1WWXXaaAgADl5OSobdu22rFjh3r06KHi4mKrIwLwQGd+/Tf13pwz+KkHI9lsNs2YMUOFhYX6/PPPlZOTo8OHD1Ny4HHKysoUGxtLybFIYWFhlRsUl5SUGP8LBoCLz/z58xUeHi4/Pz/5+fkpPDxc8+bNszqWy/CTD0arV6+eevbsqfDwcPn6+lodB3C7kSNHaunSpVbH8FhRUVFatmyZ4+Mz5WbevHmKjo62KhYAD5ScnKwxY8bouuuu05tvvqk333xT1113ncaOHWvsG8FcugYjlZSUaPr06crIyKhyN3gTr0MFqvLAAw9o8eLF6tatm7p27VppMYLU1FSLknmGtWvX6s9//rNuu+02LVy4UH/729/07bffav369fr0008VGRlpdUQAHqJJkyaaOXOmbr31Vqfxf/3rXxo9enSVG0v/0bHqGow0atQoffrpp7r99tvVvHlzLhGBx/rqq6/UvXt3SdLXX3/tdIz/Llzviiuu0ObNmzV9+nR16dJFK1euVI8ePZSZmakuXbpYHQ+ABzl58qSioqIqjUdGRurUqVMWJHI9ZnRgpPr162vZsmXq06eP1VEAAAAsN3r0aNWuXbvSTP64ceN0/PhxzZo1y6JkrsOMDozUoEEDNWzY0OoYAKDy8nK9++67+u677yRJYWFhuuGGG9g4FIDLJSUlOf5ss9k0b948rVy5Updddpkk6YsvvlBubq7i4uKsiuhSzOjASK+++qref/99LVq0SP7+/lbHAdzqpptu0sKFCxUYGKibbrrpnOe+8847bkrlmb755htdf/31ys/PV8eOHSWd3jS0SZMm+ve//63w8HCLEwIwWf/+/c/rPJvNpk8++cTFadyPt5NgpOeee047d+5UcHCwQkNDK92AnZ2dbVEywPWCgoIc998EBQVZnMazjRo1Spdeeqk2btyoBg0aSJJ++ukn3XHHHbr77ru1fv16ixMCMNmqVausjmApZnRgpCeeeOKcxydPnuymJIC1jh8/roqKCtWtW1eStGfPHr333nvq3LmzBg0aZHE689WpU0cbN27UpZde6jT+9ddfq2fPnjp+/LhFyQDAfMzowEgUGeC0G264QTfddJPuueceHTlyRJdddplq166tQ4cOKTU1Vffee6/VEY12ySWXqKCgoFLROXjwoNq3b29RKgDwDGwYCmMdOXJE8+bN08SJE3X48GFJpy9Z279/v8XJAPfJzs7WlVdeKUl66623FBwcrL1792rx4sWaOXOmxenMl5KSogceeEBvvfWW9u3bp3379umtt97Sgw8+qBkzZqi4uNjxAADULC5dg5G2bNmimJgYBQUFac+ePdq6davatm2rSZMmKTc3V4sXL7Y6IuAW/v7++v7779WqVSvdcsstuvTSSzV58mTl5eWpY8eOOnbsmNURjebl9cv7iWfumzrzY/fXH9tsNpWXl7s/IAAYjEvXYKSkpCTdcccdevrppxUQEOAYv/baazV8+HALkwHu1b59e7333nsaOnSo/vvf/2rs2LGSTl86FRgYaHE683n6jcAAYCWKDoz05Zdfavbs2ZXGW7Zsqfz8fAsSAdZITk7W8OHDNXbsWA0YMEDR0dGSpJUrV6p79+4WpzNfv379zuu8++67T5deeqkaN27s4kQA4Dm4RwdG8vX1rfKa9zP7VwCe4uabb1Zubq42btyoFStWOMYHDBig559/3sJk+LVXX32V+3QAoIZRdGCk66+/XlOmTNHJkyclnb4WPjc3V+PHj9ewYcMsTge4V7NmzdS9e3en+0V69eqlTp06WZgKv8btsgBQ8yg6MNJzzz2no0ePqmnTpjp+/Lj69eun9u3bKyAgQE899ZTV8QAAAOBi3KMDIwUFBemjjz7SunXrlJOTo6NHj6pHjx6KiYmxOhoAAADcgOWlYaTFixcrNjZWvr6+TuNlZWVasmSJ4uLiLEoGAJUFBAQoJydHbdu2tToKABiDogMjeXt764cfflDTpk2dxn/88Uc1bdqU/SoAXFQoOgBQ87hHB0Y6swHf/9q3b5+CgoIsSAQAZ3fbbbexrxEA1DDu0YFRunfvLpvNJpvNpgEDBqhWrV/+L15eXq7du3dr8ODBFiYE4Ek2bNigzMxMx/5dzZo1U3R0tHr16uV03ssvv2xFPAAwGkUHRrnxxhslSZs3b9agQYNUr149xzEfHx+FhoayvDQAlzt48KCGDRumdevWqVWrVgoODpYkFRQUaOzYserTp4/efvvtSpfXAgBqDvfowEiLFi1SbGys/Pz8rI4CwAPdfPPNOnDggP75z3+qY8eOTse2bt2qO++8Uy1atNCbb75pUUIAMB9FB0YrKyvTwYMHVVFR4TTeqlUrixIB8AQBAQH67LPP1L179yqPZ2Vl6aqrrtLPP//s5mQA4Dm4dA1G2r59u+68806tX7/eafzMIgWsugbAlXx9fVVcXHzW4z///HOl5e8BADWLogMj3XHHHapVq5Y+/PBDNW/evMoV2ADAVWJjYzVy5Eg9//zzGjBggGNFteLiYmVkZCgpKUm33nqrxSkBwGxcugYj1a1bV1lZWerUqZPVUQB4oNLSUj344INasGCBTp06JR8fH0mnL6etVauW7rrrLj3//PPM6gCAC1F0YKSePXvq+eef1xVXXGF1FAAerLi4WFlZWU7LS0dGRrJnDgC4AUUHRvrkk080adIkTZs2TV26dFHt2rWdjvNLBgAAgNkoOjCSl5eXJFW6N4fFCABcDAoKCjR79mwlJydbHQUAjEXRgZE+/fTTcx7v16+fm5IAQGU5OTnq0aMHb7oAgAux6hqMRJEBYKUtW7ac8/jWrVvdlAQAPBczOjDWmjVrNHv2bO3atUtvvvmmWrZsqVdeeUVt2rRhkQIALuXl5SWbzaaqfsSeGecyWgBwLS+rAwCu8Pbbb2vQoEGqU6eOsrOzVVpaKkkqKirStGnTLE4HwHQNGzbU3LlztXv37kqPXbt26cMPP7Q6IgAYj0vXYKSpU6cqPT1dcXFxWrJkiWO8T58+mjp1qoXJAHiCyMhIHThwQK1bt67y+JEjR6qc7QEA1ByKDoy0detW9e3bt9J4UFCQjhw54v5AADzKPffco5KSkrMeb9Wqlf75z3+6MREAeB6KDozUrFkz7dixQ6GhoU7ja9euVdu2ba0JBcBjDB069JzHGzRooJEjR7opDQB4Ju7RgZESEhI0ZswYffHFF7LZbDpw4IBee+01jRs3Tvfee6/V8QDASWBgoHbt2mV1DAAwCjM6MNKECRNUUVGhAQMG6NixY+rbt698fX01btw4jR492up4AOCE+3UAoOaxvDSMVlZWph07dujo0aMKCwtTvXr1rI4EAJUEBAQoJyeHS2sBoAZx6RqM5uPjo7CwMHXq1Ekff/yxvvvuO6sjAQAAwA0oOjDSLbfcopdeekmSdPz4cfXs2VO33HKLunbtqrffftvidAAAAHA1ig6M9Nlnn+nKK6+UJL377ruqqKjQkSNHNHPmTPbRAXDRsdlsVkcAAONQdGCkoqIiNWzYUJK0YsUKDRs2TP7+/hoyZIi2b99ucToAcMbtsgBQ8yg6MFJISIgyMzNVUlKiFStWaODAgZKkn376SX5+fhanA+CJ7Hb7WQvNf/7zH7Vs2dLNiQDAbBQdGOnBBx/UiBEj9Kc//UktWrTQVVddJen0JW1dunSxNhwAjzJ//nyFh4fLz89Pfn5+Cg8P17x585zOueKKK+Tr62tRQgAwE8tLw1hZWVnKzc3VNddc41hWetmyZapfv7769OljcToAniA5OVmpqakaPXq0oqOjJUmZmZl66aWXNHbsWE2ZMsXihABgLooOPFpgYKA2b97M3hUAXKJJkyaaOXOmbr31Vqfxf/3rXxo9erQOHTpkUTIAMB+XrsGj0fMBuNLJkycVFRVVaTwyMlKnTp2yIBEAeA6KDgAALnL77bfr5ZdfrjQ+Z84cjRgxwoJEAOA5alkdAAAAkyQlJTn+bLPZNG/ePK1cuVKXXXaZJOmLL75Qbm6u4uLirIoIAB6BogMAQA3atGmT08eRkZGSpJ07d0qSGjdurMaNG+ubb75xezYA8CQUHXg0diMHUNNWrVpldQQAgLhHBx6OxQgAAADMxIwOjHemzFQ1e8Nu5ABcqX///uecOf7kk0/cmAYAPAszOjAWu5EDsFpERIS6devmeISFhamsrEzZ2dnq0qWL1fEAwGjM6MBIZ9uNfOzYscrNzWU3cgBu8fzzz1c5/vjjj+vo0aNuTgMAnsVm5yYFGIjdyAFczHbs2KFevXrp8OHDVkcBAGNx6RqMxG7kAC5mmZmZ8vPzszoGABiNS9dgpDO7kaempjqNsxs5AHe66aabnD622+364YcftHHjRj322GMWpQIAz0DRgTHYjRzAxSYoKMjpYy8vL3Xs2FFTpkzRwIEDLUoFAJ6Be3RgjP79+5/XeTabjSVdAQAADEfRAQDAxcrKynTw4EFVVFQ4jbdq1cqiRABgPi5dAwDARbZt26a77rpL69evdxq32+2y2WwqLy+3KBkAmI+iAyOxGzmAi0F8fLxq1aqlDz/8UM2bNz/nv0sAgJpF0YGRIiIinD4+efKkNm/erK+//lojR460JhQAj7N582ZlZWWpU6dOVkcBAI9D0YGR2I0cwMUgLCyMDYoBwCIsRgCPwm7kAFytuLjY8eeNGzdq0qRJmjZtmrp06aLatWs7nRsYGOjueADgMZjRgUdhN3IArla/fn2ne3HsdrsGDBjgdA6LEQCA61F0YCR2IwdglVWrVlkdAQAgLl2DoeLj450+9vLyUpMmTXT11VezGzmAi859992nKVOmqHHjxlZHAQBjUHQAALBYYGCgNm/erLZt21odBQCMwaVrMBq7kQP4I+A9RwCoeRQdGIndyAEAADwbRQdGYjdyAAAAz0bRgZHYjRwAAMCzeVkdAHAFdiMHAADwbBQdGKO4uNjxmDFjhh555BGtXr1aP/74o9OxX+9aDgA17aabbnL8O7N48WKVlpb+5nNuu+02BQYGujoaAHgUlpeGMby8vCrtRv6/9+awGAEAV/Px8dHevXvVvHlzeXt764cfflDTpk2tjgUAHod7dGAMdiMHcDHo1KmTJk6cqP79+8tut+uNN94462xNXFycm9MBgOdgRgcejd3IAdS09evXKykpSTt37tThw4cVEBBQ5cqPNptNhw8ftiAhAHgGig48GruRA3AlLy8v7d+/X82bN3cat9vtys3NVevWrS1KBgDmYzECeDR6PgArHD58mDdYAMDFKDoAALiQt7d3pbGjR4/Kz8/PgjQA4DlYjAAAgBqWlJQk6fR9OMnJyfL393ccKy8v1xdffKGIiAiL0gGAZ6DoAABQwzZt2iTp9OWxX331lXx8fBzHfHx81K1bN40bN86qeADgESg6AADUsDPL3cfHx+uFF15gM1AAsAD36MAY7EYO4GLzz3/+k39jAMAiLC8NY7AbOQAAAM7g0jUYg93IAQAAcAYzOjAGu5EDAADgDIoOjMRu5AAAAJ6NxQjgUdiNHAAAwDNQdGAsdiMHAADwXCxGAKOwGzkAAAAkig4Mw27kAAAAkFiMAIZiN3IAAADPRtEBAAAAYBwWIwAAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjPP/AezNF5HQpXjoAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "baseline_pred_probs = np.max(model_0.predict_proba(val_sentences), axis = 1)\n",
        "combined_pred_probs = baseline_pred_probs + tf.squeeze(model_2_pred_probs, axis = 1) + tf.squeeze(model_6_pred_probs)\n",
        "combined_preds = tf.round(combined_pred_probs / 3)\n",
        "combined_preds[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qzsc0GRCiIeG",
        "outputId": "a1e6680b-ac06-4f3a-b5e9-d0daa2152a5b"
      },
      "execution_count": 98,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10,), dtype=float32, numpy=array([0., 1., 1., 0., 0., 1., 1., 1., 1., 0.], dtype=float32)>"
            ]
          },
          "metadata": {},
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ensemble_results = calculate_results(val_labels, combined_preds)\n",
        "ensemble_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8F4Wng7TFc3D",
        "outputId": "89856ca8-f316-4cee-864b-40e85575e21b"
      },
      "execution_count": 99,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 79.00262467191601,\n",
              " 'precision': 0.789737110325998,\n",
              " 'recall': 0.7900262467191601,\n",
              " 'f1': 0.7896841842138155}"
            ]
          },
          "metadata": {},
          "execution_count": 99
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "all_model_results.loc['ensemble_results'] = ensemble_results"
      ],
      "metadata": {
        "id": "bK6UAB0BFjMv"
      },
      "execution_count": 100,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "all_model_results.loc['ensemble_results']['accuracy'] = all_model_results.loc['ensemble_results']['accuracy']/100"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LG7Jyq4TFrBB",
        "outputId": "57fa58ac-c8c1-4afb-dfa1-85b32ca6834d"
      },
      "execution_count": 103,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-103-4986557ac854>:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  all_model_results.loc['ensemble_results']['accuracy'] = all_model_results.loc['ensemble_results']['accuracy']/100\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "all_model_results"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 331
        },
        "id": "XWq4PVyYGBbj",
        "outputId": "4854f8e2-ff42-445e-cb4a-7cfaab53042e"
      },
      "execution_count": 104,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                          accuracy  precision    recall        f1\n",
              "baseline                  0.792651   0.811139  0.792651  0.786219\n",
              "simple_dense              0.780840   0.784127  0.780840  0.778263\n",
              "lstm                      0.762467   0.764803  0.762467  0.759823\n",
              "gru                       0.770341   0.772938  0.770341  0.767784\n",
              "bidirectional             0.751969   0.752786  0.751969  0.749925\n",
              "conv1d                    0.812336   0.814569  0.812336  0.810789\n",
              "tf_hub_sentence_encoder   0.812336   0.814569  0.812336  0.810789\n",
              "tf_hub_10_percent_data    0.778215   0.783827  0.778215  0.774705\n",
              "ensemble_results         79.002625   0.789737  0.790026  0.789684"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-052a38ab-de9e-41e9-b949-3b70749da213\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>accuracy</th>\n",
              "      <th>precision</th>\n",
              "      <th>recall</th>\n",
              "      <th>f1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>baseline</th>\n",
              "      <td>0.792651</td>\n",
              "      <td>0.811139</td>\n",
              "      <td>0.792651</td>\n",
              "      <td>0.786219</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>simple_dense</th>\n",
              "      <td>0.780840</td>\n",
              "      <td>0.784127</td>\n",
              "      <td>0.780840</td>\n",
              "      <td>0.778263</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>lstm</th>\n",
              "      <td>0.762467</td>\n",
              "      <td>0.764803</td>\n",
              "      <td>0.762467</td>\n",
              "      <td>0.759823</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>gru</th>\n",
              "      <td>0.770341</td>\n",
              "      <td>0.772938</td>\n",
              "      <td>0.770341</td>\n",
              "      <td>0.767784</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>bidirectional</th>\n",
              "      <td>0.751969</td>\n",
              "      <td>0.752786</td>\n",
              "      <td>0.751969</td>\n",
              "      <td>0.749925</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>conv1d</th>\n",
              "      <td>0.812336</td>\n",
              "      <td>0.814569</td>\n",
              "      <td>0.812336</td>\n",
              "      <td>0.810789</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>tf_hub_sentence_encoder</th>\n",
              "      <td>0.812336</td>\n",
              "      <td>0.814569</td>\n",
              "      <td>0.812336</td>\n",
              "      <td>0.810789</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>tf_hub_10_percent_data</th>\n",
              "      <td>0.778215</td>\n",
              "      <td>0.783827</td>\n",
              "      <td>0.778215</td>\n",
              "      <td>0.774705</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ensemble_results</th>\n",
              "      <td>79.002625</td>\n",
              "      <td>0.789737</td>\n",
              "      <td>0.790026</td>\n",
              "      <td>0.789684</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-052a38ab-de9e-41e9-b949-3b70749da213')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-052a38ab-de9e-41e9-b949-3b70749da213 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-052a38ab-de9e-41e9-b949-3b70749da213');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-08edb350-cdf4-4747-b6e4-56a9bfa65d00\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-08edb350-cdf4-4747-b6e4-56a9bfa65d00')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-08edb350-cdf4-4747-b6e4-56a9bfa65d00 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_8aa00db2-ae64-46ed-bb2c-17ce299224d4\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('all_model_results')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_8aa00db2-ae64-46ed-bb2c-17ce299224d4 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('all_model_results');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "all_model_results",
              "summary": "{\n  \"name\": \"all_model_results\",\n  \"rows\": 9,\n  \"fields\": [\n    {\n      \"column\": \"accuracy\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 26.07333488741847,\n        \"min\": 0.7519685039370079,\n        \"max\": 79.00262467191601,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          0.7808398950131233,\n          0.8123359580052494,\n          0.7926509186351706\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"precision\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.0223222451558682,\n        \"min\": 0.7527859023002904,\n        \"max\": 0.8145687227271431,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          0.7841274438015813,\n          0.8145687227271431,\n          0.8111390004213173\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"recall\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.020718705254392317,\n        \"min\": 0.7519685039370079,\n        \"max\": 0.8123359580052494,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          0.7808398950131233,\n          0.8123359580052494,\n          0.7926509186351706\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"f1\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.020956227016283198,\n        \"min\": 0.7499245530786587,\n        \"max\": 0.8107894766679798,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          0.7782630349987989,\n          0.8107894766679798,\n          0.7862189758049549\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_6.save('model_6.h5')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ag2nFmkBGVkf",
        "outputId": "567c42a9-2c22-453f-b8ae-f2aba6b5dd20"
      },
      "execution_count": 105,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/tf_keras/src/engine/training.py:3098: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native TF-Keras format, e.g. `model.save('my_model.keras')`.\n",
            "  saving_api.save_model(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "loaded_model_6 = tf_keras.models.load_model('model_6.h5', custom_objects = {'KerasLayer' : hub.KerasLayer})"
      ],
      "metadata": {
        "id": "OzqzEmF4Ggla"
      },
      "execution_count": 107,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loaded_model_6.evaluate(val_sentences, val_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KGR6MamFGtoR",
        "outputId": "f755feda-209b-46f9-c7a6-4b72b0a0558c"
      },
      "execution_count": 108,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 1s 12ms/step - loss: 0.4316 - accuracy: 0.8123\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.4315696954727173, 0.8123359680175781]"
            ]
          },
          "metadata": {},
          "execution_count": 108
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model_6.save('model_6_SavedModel_format')"
      ],
      "metadata": {
        "id": "HDYiWbzjHwYs"
      },
      "execution_count": 109,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loaded_model_6_SavedModel = tf_keras.models.load_model('model_6_SavedModel_format')"
      ],
      "metadata": {
        "id": "lkFTpQ2aH13e"
      },
      "execution_count": 111,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loaded_model_6_SavedModel.evaluate(val_sentences, val_labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dHyiF-6FIFne",
        "outputId": "cc185e16-0002-4606-f9b6-bbbd38116c07"
      },
      "execution_count": 112,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 1s 18ms/step - loss: 0.4316 - accuracy: 0.8123\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.4315696954727173, 0.8123359680175781]"
            ]
          },
          "metadata": {},
          "execution_count": 112
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "val_df = pd.DataFrame({\n",
        "    'text' : val_sentences,\n",
        "    'target' : val_labels,\n",
        "    'pred' : model_6_preds,\n",
        "    'pred_prob' : tf.squeeze(model_6_pred_probs)\n",
        "})\n",
        "val_df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "F8mj4gC4Ig41",
        "outputId": "9f242fde-fb92-4049-bf57-e0d68a1ba6bf"
      },
      "execution_count": 113,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                text  target  pred  pred_prob\n",
              "0  DFR EP016 Monthly Meltdown - On Dnbheaven 2015...       0   0.0   0.144270\n",
              "1  FedEx no longer to transport bioterror germs i...       0   1.0   0.731866\n",
              "2  Gunmen kill four in El Salvador bus attack: Su...       1   1.0   0.984290\n",
              "3  @camilacabello97 Internally and externally scr...       1   0.0   0.190733\n",
              "4  Radiation emergency #preparedness starts with ...       1   1.0   0.747906"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b5db6743-90b7-41a1-a7b2-57810a5b5151\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "      <th>pred</th>\n",
              "      <th>pred_prob</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>DFR EP016 Monthly Meltdown - On Dnbheaven 2015...</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.144270</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>FedEx no longer to transport bioterror germs i...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.731866</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Gunmen kill four in El Salvador bus attack: Su...</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.984290</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>@camilacabello97 Internally and externally scr...</td>\n",
              "      <td>1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.190733</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Radiation emergency #preparedness starts with ...</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.747906</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b5db6743-90b7-41a1-a7b2-57810a5b5151')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-b5db6743-90b7-41a1-a7b2-57810a5b5151 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-b5db6743-90b7-41a1-a7b2-57810a5b5151');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-88326bd4-4af0-4a11-92ee-d9420d0e8e2b\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-88326bd4-4af0-4a11-92ee-d9420d0e8e2b')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-88326bd4-4af0-4a11-92ee-d9420d0e8e2b button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "val_df",
              "summary": "{\n  \"name\": \"val_df\",\n  \"rows\": 762,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 761,\n        \"samples\": [\n          \"collapsed the moment i got home last night lol\",\n          \"Reality Training: Train falls off elevated tracks during windstorm http://t.co/qzRciNaF5z\",\n          \"Storm headed towards Idaho Falls with blowing dust &amp; winds to 60 mph.  US HWY 20 &amp; I15 look out.  #idwx http://t.co/0cR74m1Uxm\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"target\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          1,\n          0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pred\",\n      \"properties\": {\n        \"dtype\": \"float32\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          1.0,\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pred_prob\",\n      \"properties\": {\n        \"dtype\": \"float32\",\n        \"num_unique_values\": 761,\n        \"samples\": [\n          0.12611110508441925,\n          0.6912522912025452\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 113
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "most_wrong = val_df[val_df['target'] != val_df['pred']].sort_values('pred_prob', ascending = False)\n",
        "most_wrong[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "iF79p2diIvQu",
        "outputId": "03c77c6a-76a9-4064-ff5b-8b6551b3260a"
      },
      "execution_count": 114,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  text  target  pred  \\\n",
              "31   ? High Skies - Burning Buildings ? http://t.co...       0   1.0   \n",
              "759  FedEx will no longer transport bioterror patho...       0   1.0   \n",
              "628  @noah_anyname That's where the concentration c...       0   1.0   \n",
              "393  @SonofLiberty357 all illuminated by the bright...       0   1.0   \n",
              "49   @madonnamking RSPCA site multiple 7 story high...       0   1.0   \n",
              "209  Ashes 2015: AustraliaÛªs collapse at Trent Br...       0   1.0   \n",
              "251  @AshGhebranious civil rights continued in the ...       0   1.0   \n",
              "109  [55436] 1950 LIONEL TRAINS SMOKE LOCOMOTIVES W...       0   1.0   \n",
              "144                                 The Sound of Arson       0   1.0   \n",
              "567  @RebeccaforReal accepts Wisconsin Emergency Re...       0   1.0   \n",
              "\n",
              "     pred_prob  \n",
              "31    0.906667  \n",
              "759   0.848571  \n",
              "628   0.837095  \n",
              "393   0.835707  \n",
              "49    0.833949  \n",
              "209   0.817409  \n",
              "251   0.773691  \n",
              "109   0.773263  \n",
              "144   0.767427  \n",
              "567   0.766256  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-69de578c-44b9-4470-b31e-ed8e339a3925\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "      <th>pred</th>\n",
              "      <th>pred_prob</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>? High Skies - Burning Buildings ? http://t.co...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.906667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>759</th>\n",
              "      <td>FedEx will no longer transport bioterror patho...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.848571</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>628</th>\n",
              "      <td>@noah_anyname That's where the concentration c...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.837095</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>393</th>\n",
              "      <td>@SonofLiberty357 all illuminated by the bright...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.835707</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>@madonnamking RSPCA site multiple 7 story high...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.833949</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>209</th>\n",
              "      <td>Ashes 2015: AustraliaÛªs collapse at Trent Br...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.817409</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>251</th>\n",
              "      <td>@AshGhebranious civil rights continued in the ...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.773691</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>109</th>\n",
              "      <td>[55436] 1950 LIONEL TRAINS SMOKE LOCOMOTIVES W...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.773263</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>144</th>\n",
              "      <td>The Sound of Arson</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.767427</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>567</th>\n",
              "      <td>@RebeccaforReal accepts Wisconsin Emergency Re...</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.766256</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-69de578c-44b9-4470-b31e-ed8e339a3925')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-69de578c-44b9-4470-b31e-ed8e339a3925 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-69de578c-44b9-4470-b31e-ed8e339a3925');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-2b833df3-c468-4abe-b3fa-d8e94afc7375\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-2b833df3-c468-4abe-b3fa-d8e94afc7375')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-2b833df3-c468-4abe-b3fa-d8e94afc7375 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"most_wrong[:10]\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"The Sound of Arson\",\n          \"FedEx will no longer transport bioterror pathogens in wake of anthrax lab mishaps http://t.co/lHpgxc4b8J\",\n          \"Ashes 2015: Australia\\u0089\\u00db\\u00aas collapse at Trent Bridge among worst in history: England bundled out Australia for 60 ... http://t.co/t5TrhjUAU0\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"target\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 0,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pred\",\n      \"properties\": {\n        \"dtype\": \"float32\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          1.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pred_prob\",\n      \"properties\": {\n        \"dtype\": \"float32\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          0.7674270272254944\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 114
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for row in most_wrong[:10].itertuples():\n",
        "  _, text, target, pred, prob = row\n",
        "  print(f'Target : {target}, Pred : {pred}, Prob : {prob}')\n",
        "  print(f'Text:\\n{text}\\n')\n",
        "  print('---\\n')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "htyrbTnVJQKg",
        "outputId": "321d331c-b899-4324-bcc3-de625808bf9e"
      },
      "execution_count": 115,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Target : 0, Pred : 1.0, Prob : 0.9066672325134277\n",
            "Text:\n",
            "? High Skies - Burning Buildings ? http://t.co/uVq41i3Kx2 #nowplaying\n",
            "\n",
            "---\n",
            "\n",
            "Target : 0, Pred : 1.0, Prob : 0.8485706448554993\n",
            "Text:\n",
            "FedEx will no longer transport bioterror pathogens in wake of anthrax lab mishaps http://t.co/lHpgxc4b8J\n",
            "\n",
            "---\n",
            "\n",
            "Target : 0, Pred : 1.0, Prob : 0.8370948433876038\n",
            "Text:\n",
            "@noah_anyname That's where the concentration camps and mass murder come in. \n",
            " \n",
            "EVERY. FUCKING. TIME.\n",
            "\n",
            "---\n",
            "\n",
            "Target : 0, Pred : 1.0, Prob : 0.8357065916061401\n",
            "Text:\n",
            "@SonofLiberty357 all illuminated by the brightly burning buildings all around the town!\n",
            "\n",
            "---\n",
            "\n",
            "Target : 0, Pred : 1.0, Prob : 0.8339487314224243\n",
            "Text:\n",
            "@madonnamking RSPCA site multiple 7 story high rise buildings next to low density character residential in an area that floods\n",
            "\n",
            "---\n",
            "\n",
            "Target : 0, Pred : 1.0, Prob : 0.8174088001251221\n",
            "Text:\n",
            "Ashes 2015: AustraliaÛªs collapse at Trent Bridge among worst in history: England bundled out Australia for 60 ... http://t.co/t5TrhjUAU0\n",
            "\n",
            "---\n",
            "\n",
            "Target : 0, Pred : 1.0, Prob : 0.7736914753913879\n",
            "Text:\n",
            "@AshGhebranious civil rights continued in the 60s. And what about trans-generational trauma? if anything we should listen to the Americans.\n",
            "\n",
            "---\n",
            "\n",
            "Target : 0, Pred : 1.0, Prob : 0.7732632160186768\n",
            "Text:\n",
            "[55436] 1950 LIONEL TRAINS SMOKE LOCOMOTIVES WITH MAGNE-TRACTION INSTRUCTIONS http://t.co/xEZBs3sq0y http://t.co/C2x0QoKGlY\n",
            "\n",
            "---\n",
            "\n",
            "Target : 0, Pred : 1.0, Prob : 0.7674270272254944\n",
            "Text:\n",
            "The Sound of Arson\n",
            "\n",
            "---\n",
            "\n",
            "Target : 0, Pred : 1.0, Prob : 0.7662561535835266\n",
            "Text:\n",
            "@RebeccaforReal accepts Wisconsin Emergency Response Plan on behalf of @GovWalker #nbc15 http://t.co/Pis0aiVRbR\n",
            "\n",
            "---\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for row in most_wrong[-10:].itertuples():\n",
        "  _, text, target, pred, prob = row\n",
        "  print(f'Target : {target}, Pred : {pred}, Prob : {prob}')\n",
        "  print(f'Text:\\n{text}\\n')\n",
        "  print('---\\n')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4eSYJTJiKA3N",
        "outputId": "31a3e64c-8c80-40bb-8a91-a11bb99c9bac"
      },
      "execution_count": 117,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Target : 1, Pred : 0.0, Prob : 0.05960942432284355\n",
            "Text:\n",
            "'The way you move is like a full on rainstorm and I'm a house of cards'\n",
            "\n",
            "---\n",
            "\n",
            "Target : 1, Pred : 0.0, Prob : 0.057065654546022415\n",
            "Text:\n",
            "going to redo my nails and watch behind the scenes of desolation of smaug ayyy\n",
            "\n",
            "---\n",
            "\n",
            "Target : 1, Pred : 0.0, Prob : 0.05468815565109253\n",
            "Text:\n",
            "Lucas Duda is Ghost Rider. Not the Nic Cage version but an actual 'engulfed in flames' badass. #Mets\n",
            "\n",
            "---\n",
            "\n",
            "Target : 1, Pred : 0.0, Prob : 0.04879409447312355\n",
            "Text:\n",
            "You can never escape me. Bullets don't harm me. Nothing harms me. But I know pain. I know pain. Sometimes I share it. With someone like you.\n",
            "\n",
            "---\n",
            "\n",
            "Target : 1, Pred : 0.0, Prob : 0.04730147868394852\n",
            "Text:\n",
            "@willienelson We need help! Horses will die!Please RT &amp; sign petition!Take a stand &amp; be a voice for them! #gilbert23 https://t.co/e8dl1lNCVu\n",
            "\n",
            "---\n",
            "\n",
            "Target : 1, Pred : 0.0, Prob : 0.04103677719831467\n",
            "Text:\n",
            "I get to smoke my shit in peace\n",
            "\n",
            "---\n",
            "\n",
            "Target : 1, Pred : 0.0, Prob : 0.0379406176507473\n",
            "Text:\n",
            "@SoonerMagic_ I mean I'm a fan but I don't need a girl sounding off like a damn siren\n",
            "\n",
            "---\n",
            "\n",
            "Target : 1, Pred : 0.0, Prob : 0.03625212609767914\n",
            "Text:\n",
            "Reddit Will Now QuarantineÛ_ http://t.co/pkUAMXw6pm #onlinecommunities #reddit #amageddon #freespeech #Business http://t.co/PAWvNJ4sAP\n",
            "\n",
            "---\n",
            "\n",
            "Target : 1, Pred : 0.0, Prob : 0.03369549661874771\n",
            "Text:\n",
            "Why are you deluged with low self-image? Take the quiz: http://t.co/XsPqdOrIqj http://t.co/CQYvFR4UCy\n",
            "\n",
            "---\n",
            "\n",
            "Target : 1, Pred : 0.0, Prob : 0.029774917289614677\n",
            "Text:\n",
            "Ron &amp; Fez - Dave's High School Crush https://t.co/aN3W16c8F6 via @YouTube\n",
            "\n",
            "---\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_sentences = test_df['text'].tolist()\n",
        "test_samples = random.sample(test_sentences, 10)\n",
        "for test_sample in test_samples:\n",
        "  pred_prob = tf.squeeze(model_6.predict([test_sample]))\n",
        "  pred = tf.round(pred_prob)\n",
        "  print(f\"Pred: {int(pred)}, Prob: {pred_prob}\")\n",
        "  print(f\"Text:\\n{test_sample}\\n\")\n",
        "  print(\"----\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2r-TNbYxKK3-",
        "outputId": "8d40e902-28f4-45c2-e480-263e47d73b32"
      },
      "execution_count": 120,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 114ms/step\n",
            "Pred: 1, Prob: 0.7329937219619751\n",
            "Text:\n",
            "119 Sri Lanka's 'Emergency' Line. \n",
            "Busy for the past 30 mins. \n",
            "Suppose I need to plan my emergencies better. #SL #RT\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 81ms/step\n",
            "Pred: 0, Prob: 0.24857142567634583\n",
            "Text:\n",
            "@OpTic_DKarma dude they demolished you!\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 79ms/step\n",
            "Pred: 1, Prob: 0.6256638765335083\n",
            "Text:\n",
            "#Turkey invades #Israel - Halfway to #Armageddon http://t.co/xUOh3sJNXF\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 68ms/step\n",
            "Pred: 0, Prob: 0.22302860021591187\n",
            "Text:\n",
            "Permutable site conspiracy up-to-the-minute upheaval: QWkD\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 76ms/step\n",
            "Pred: 0, Prob: 0.05995811149477959\n",
            "Text:\n",
            "@ModCloth having some trouble with your site- keep getting this every time I log in? http://t.co/dnKkYiFsvT\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 71ms/step\n",
            "Pred: 0, Prob: 0.040715593844652176\n",
            "Text:\n",
            "the bitches say im hot i say no bitch im blazing\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 78ms/step\n",
            "Pred: 0, Prob: 0.022453051060438156\n",
            "Text:\n",
            "The Devil Wears Prada is still one of my favourite films ??\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 77ms/step\n",
            "Pred: 1, Prob: 0.5572730302810669\n",
            "Text:\n",
            "@Jinxy1888 Genius economist! Let's double pay of all emergency services military teachers etc to that tube drivers #PayForItHow #tubestrike\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 75ms/step\n",
            "Pred: 1, Prob: 0.842538595199585\n",
            "Text:\n",
            "(SJ GIST): 148 Houses Farm Produce Destroyed By Flood In Sokoto: About 148 houses were on Saturday destroyed ... http://t.co/vqU1Y31hKU\n",
            "\n",
            "----\n",
            "\n",
            "1/1 [==============================] - 0s 72ms/step\n",
            "Pred: 1, Prob: 0.885055422782898\n",
            "Text:\n",
            "California wild fires...clearly caused by global climate change and excess CO2!!!?\n",
            "\n",
            "----\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sample_tweet = 'Life like an ensemble: take the best choices from others and make your own'"
      ],
      "metadata": {
        "id": "RLY9tD8SKfhH"
      },
      "execution_count": 121,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def predict_on_sentence(model, sentence):\n",
        "  pred_prob = model.predict([sentence])\n",
        "  pred_label = tf.squeeze(tf.round(pred_prob)).numpy()\n",
        "  print(f\"Pred: {pred_label}\", \"(real disaster)\" if pred_label > 0 else \"(not real disaster)\", f\"Prob: {pred_prob[0][0]}\")\n",
        "  print(f\"Text:\\n{sentence}\")"
      ],
      "metadata": {
        "id": "O5t1T44sLMmD"
      },
      "execution_count": 122,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "predict_on_sentence(model_6, sample_tweet)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CjE7BCEILf74",
        "outputId": "7cb30a22-69e7-44ee-a543-d4b36fa6c7bb"
      },
      "execution_count": 123,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 253ms/step\n",
            "Pred: 0.0 (not real disaster) Prob: 0.039919137954711914\n",
            "Text:\n",
            "Life like an ensemble: take the best choices from others and make your own\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "beirut_tweet_1 = \"Reports that the smoke in Beirut sky contains nitric acid, which is toxic. Please share and refrain from stepping outside unless urgent. #Lebanon\"\n",
        "beirut_tweet_2 = \"#Beirut declared a “devastated city”, two-week state of emergency officially declared. #Lebanon\""
      ],
      "metadata": {
        "id": "em2NtiFhLkPC"
      },
      "execution_count": 124,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "predict_on_sentence(model_6, beirut_tweet_1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jU6r1NbALqCJ",
        "outputId": "0e2a8586-380a-41bf-d25c-82eec1d8194d"
      },
      "execution_count": 125,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 175ms/step\n",
            "Pred: 1.0 (real disaster) Prob: 0.9545288681983948\n",
            "Text:\n",
            "Reports that the smoke in Beirut sky contains nitric acid, which is toxic. Please share and refrain from stepping outside unless urgent. #Lebanon\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predict_on_sentence(model_6, beirut_tweet_2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m-EVuhVALse9",
        "outputId": "6d610ae1-2cd6-4c01-c549-6d81837041d6"
      },
      "execution_count": 126,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 89ms/step\n",
            "Pred: 1.0 (real disaster) Prob: 0.9663898348808289\n",
            "Text:\n",
            "#Beirut declared a “devastated city”, two-week state of emergency officially declared. #Lebanon\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import time\n",
        "\n",
        "def pred_timer(model, samples):\n",
        "  start_time = time.perf_counter()\n",
        "  model.predict(samples)\n",
        "  end_time = time.perf_counter()\n",
        "  total_time = end_time - start_time\n",
        "  time_per_pred = total_time / len(val_sentences)\n",
        "  return total_time, time_per_pred"
      ],
      "metadata": {
        "id": "d-tMZ0swLuKx"
      },
      "execution_count": 127,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_6_total_pred_time, model_6_time_per_pred = pred_timer(model_6, val_sentences)\n",
        "model_6_total_pred_time, model_6_time_per_pred"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_qkyYw2-MgWZ",
        "outputId": "ec8dc3c0-3c02-4ad7-fd8a-477a0cfdae88"
      },
      "execution_count": 128,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "24/24 [==============================] - 0s 17ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.841545517999748, 0.001104390443569223)"
            ]
          },
          "metadata": {},
          "execution_count": 128
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "baseline_model_total_pred_time, baseline_model_time_per_pred = pred_timer(model_0, val_sentences)\n",
        "baseline_model_total_pred_time, baseline_model_time_per_pred"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t57GrBZmMvy7",
        "outputId": "c9ee11ef-5327-48e8-d905-18607ee52d04"
      },
      "execution_count": 129,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.06668831400020281, 8.751747244121104e-05)"
            ]
          },
          "metadata": {},
          "execution_count": 129
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.figure(figsize = (10, 7))\n",
        "plt.scatter(baseline_model_time_per_pred, baseline_results['f1'], label = 'baseline')\n",
        "plt.scatter(model_6_time_per_pred, model_6_results['f1'], label = 'tf_hub_sentence_encoder')\n",
        "plt.legend()\n",
        "plt.title('F1-score versus time per prediction')\n",
        "plt.xlabel('Time per prediction')\n",
        "plt.ylabel('F1 - score')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 658
        },
        "id": "sojVPQFuM1g-",
        "outputId": "b0896d75-895f-4252-d371-b150ff64d637"
      },
      "execution_count": 132,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0, 0.5, 'F1 - score')"
            ]
          },
          "metadata": {},
          "execution_count": 132
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x700 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA18AAAJwCAYAAAB2y9s+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABhq0lEQVR4nO3deZxOdeP/8fc1M2bfLGMWjRkhS9k1kyWUyaDbciOismS5iSzTwnSHVCi5pYTuSkiURGmR7G7Jvq+DQagxtsxkMMPM5/eHn+vbZRYzjDOD1/PxuB5cn/M5n+VcZ2rezjmfy2aMMQIAAAAA3FJOBT0AAAAAALgbEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAyKOuXbsqPDy8oIeBa6xYsUI2m00rVqywl+X3ZzVt2jTZbDYdPnw439oEcPcgfAG4a139JSqr15AhQ+z1Fi1apO7du+uBBx6Qs7Mzv3TfJf744w+99tpr2rp1a0EPBQVg1KhR+vbbbwt6GADuMC4FPQAAKGivv/66ypQp41D2wAMP2P8+a9YszZ49WzVr1lRISIjVw0MB+eOPPzRixAiFh4erevXqDts+/vhjZWRkFMzAkCc3+lmNGjVK7dq1U+vWrR3Kn3nmGT355JNyc3PLpxECuJsQvgDc9Zo1a6batWtnu33UqFH6+OOPVaRIEf3jH//Qzp07LRxd/khJSZGXl1dBD+O6bpdxFilSpKCHUCBu1eeTkZGhtLQ0ubu753vb+f1ZOTs7y9nZOV/bBHD34LZDALiOkJCQm/oF7q+//tLAgQMVHh4uNzc3lSxZUo899pg2b97sUG/dunVq3ry5ihYtKi8vL1WtWlXvvfeeQ51ly5bp4YcflpeXl/z9/dWqVSvt2bPHoc5rr70mm82m3bt3q1OnTipatKjq169v3/7555+rVq1a8vDwULFixfTkk0/q6NGjOc7h66+/ls1m08qVKzNt++9//yubzeYQSvfu3at27dqpWLFicnd3V+3atfXdd9857Hf1ts+VK1fqueeeU8mSJXXPPffk+piFh4era9eumcbTqFEjNWrUyKFswoQJuv/+++Xp6amiRYuqdu3amjVrVrbzXbFihR588EFJUrdu3ey3o06bNk1S5ueIDh8+LJvNprFjx2rixIm699575enpqSZNmujo0aMyxuiNN97QPffcIw8PD7Vq1UpnzpzJ1O9PP/1k/3x9fHz0+OOPa9euXdmO89pj+b///U//+te/VLx4cfn6+qpz5876888/b6ifrl27ytvbW/Hx8WrevLl8fHz01FNPZTuGq+fd3r171b59e/n6+qp48eIaMGCALl686FDXZrOpX79+mjlzpu6//365ublp4cKFkqTff/9dzz77rAIDA+Xm5qb7779fn376aab+jh07ptatW8vLy0slS5bUoEGDlJqamqleVs98ZWRk6L333lOVKlXk7u6ugIAANW3aVBs3brSPLyUlRdOnT7d/9lfPteye+Zo0aZJ9LiEhIerbt6/Onj3rUKdRo0Z64IEHtHv3bj3yyCPy9PRUqVKlNGbMmGyPK4A7C1e+ANz1kpKSdOrUKYeyEiVK5Fv7vXv31tdff61+/fqpcuXKOn36tH755Rft2bNHNWvWlCQtXrxY//jHPxQcHKwBAwYoKChIe/bs0Q8//KABAwZIkpYsWaJmzZrp3nvv1WuvvaYLFy5owoQJqlevnjZv3pzpF8wnnnhC5cuX16hRo2SMkSSNHDlSQ4cOVfv27dWjRw+dPHlSEyZMUIMGDbRlyxb5+/tnOYfHH39c3t7e+uqrr9SwYUOHbbNnz9b9999vv1Vz165dqlevnkqVKqUhQ4bIy8tLX331lVq3bq25c+fqn//8p8P+zz33nAICAjRs2DClpKTk+pjl1scff6z+/furXbt29iCwfft2rVu3Tp06dcpyn0qVKun111/XsGHD1KtXLz388MOSpLp16+bY18yZM5WWlqbnn39eZ86c0ZgxY9S+fXs9+uijWrFihQYPHqwDBw5owoQJevHFFx1CxYwZM9SlSxdFR0fr7bff1vnz5zV58mTVr19fW7ZsydWzhv369ZO/v79ee+01xcXFafLkyfrtt9/sC1HktZ/Lly8rOjpa9evX19ixY+Xp6XndMbRv317h4eEaPXq01q5dq/fff19//vmnPvvsM4d6y5Yt01dffaV+/fqpRIkSCg8PV2Jioh566CF7OAsICNBPP/2k7t27Kzk5WQMHDpQkXbhwQY0bN9aRI0fUv39/hYSEaMaMGVq2bNl1xydJ3bt317Rp09SsWTP16NFDly9f1qpVq7R27VrVrl1bM2bMUI8ePRQREaFevXpJksqWLZtte6+99ppGjBihqKgo9enTx37sN2zYoNWrVzv8482ff/6ppk2bqk2bNmrfvr2+/vprDR48WFWqVFGzZs1yNX4AtzEDAHepqVOnGklZvrLz+OOPm7CwsDz14+fnZ/r27Zvt9suXL5syZcqYsLAw8+effzpsy8jIsP+9evXqpmTJkub06dP2sm3bthknJyfTuXNne9nw4cONJNOxY0eHtg4fPmycnZ3NyJEjHcp37NhhXFxcMpVfq2PHjqZkyZLm8uXL9rKEhATj5ORkXn/9dXtZ48aNTZUqVczFixcd5lG3bl1Tvnx5e9nV41+/fn2HNo25/jEzxpiwsDDTpUuXTOUNGzY0DRs2tL9v1aqVuf/++3NsKysbNmwwkszUqVMzbevSpYvDeXDo0CEjyQQEBJizZ8/ay2NjY40kU61aNXPp0iV7eceOHY2rq6v9GP3111/G39/f9OzZ06Gf48ePGz8/v0zl17p6LGvVqmXS0tLs5WPGjDGSzPz58/PcT5cuXYwkM2TIkBz7vurqedeyZUuH8ueee85IMtu2bbOXSTJOTk5m165dDnW7d+9ugoODzalTpxzKn3zySePn52fOnz9vjDFm/PjxRpL56quv7HVSUlJMuXLljCSzfPlyh3n8/bNatmyZkWT69++faQ5//3nz8vLK8vy6eqwPHTpkjDHmxIkTxtXV1TRp0sSkp6fb633wwQdGkvn000/tZQ0bNjSSzGeffWYvS01NNUFBQaZt27aZ+gJw5+G2QwB3vYkTJ2rx4sUOr/zk7++vdevW6Y8//shy+5YtW3To0CENHDgw05Wnq1crEhIStHXrVnXt2lXFihWzb69ataoee+wxLViwIFO7vXv3dng/b948ZWRkqH379jp16pT9FRQUpPLly2v58uU5zqNDhw46ceKEwzLeX3/9tTIyMtShQwdJ0pkzZ7Rs2TK1b99ef/31l72P06dPKzo6Wvv379fvv//u0G7Pnj0zPUNzvWOWF/7+/jp27Jg2bNhw021dzxNPPCE/Pz/7+8jISEnS008/LRcXF4fytLQ0+7FYvHixzp49q44dOzp8Ns7OzoqMjLzuZ3NVr169HK6y9OnTRy4uLvbz40b66dOnT56OQd++fR3eP//885KU6Rxt2LChKleubH9vjNHcuXPVokULGWMcxhcdHa2kpCT7bacLFixQcHCw2rVrZ9/f09PTfpUqJ3PnzpXNZtPw4cMzbbv685YXS5YsUVpamgYOHCgnp//7tapnz57y9fXVjz/+6FDf29tbTz/9tP29q6urIiIidPDgwTz3DeD2w22HAO56EREROS64kRvp6ek6efKkQ1mxYsXk6uqqMWPGqEuXLgoNDVWtWrXUvHlzde7cWffee68kKT4+XpLjCovX+u233yRJFSpUyLStUqVK+vnnnzMthnDtCo779++XMUbly5fPso/rPdfWtGlT+fn5afbs2WrcuLGkK7ccVq9eXffdd58k6cCBAzLGaOjQoRo6dGiW7Zw4cUKlSpXKdpySrnvM8mLw4MFasmSJIiIiVK5cOTVp0kSdOnVSvXr18tzW9ZQuXdrh/dUgFhoammX51eex9u/fL0l69NFHs2zX19c3V/1f+9l6e3srODjY/nxSXvtxcXGxP4eXW9eOoWzZsnJycsr0jNS1n/vJkyd19uxZffTRR/roo4+ybPvEiROSrvw8lCtXLlNYyurn41rx8fEKCQlx+EeMm5Hdz6arq6vuvfde+/ar7rnnnkzjLlq0qLZv354v4wFQuBG+ACAfHD16NNMvk8uXL1ejRo3Uvn17Pfzww/rmm2+0aNEivfPOO3r77bc1b968W/qMh4eHh8P7jIwM2Ww2/fTTT1mu1ubt7Z1je25ubmrdurW++eYbTZo0SYmJiVq9erVGjRrl0Ickvfjii4qOjs6ynXLlyuU4Tkm5OmbZXaVIT093mF+lSpUUFxenH374QQsXLtTcuXM1adIkDRs2TCNGjMhxznmV3Sp42ZWb//8s3tXjNmPGDAUFBWWq9/erZjcjr/24ubk5XM25Edl9Tlmdn9KVq4RdunTJcp+qVave1FgKg+udCwDubIQvAMgHQUFBmW5XrFatmv3vwcHBeu655/Tcc8/pxIkTqlmzpkaOHKlmzZrZH+TfuXOnoqKismw/LCxMkhQXF5dp2969e1WiRInrLgFetmxZGWNUpkwZ+5WqvOrQoYOmT5+upUuXas+ePTLG2G85lGS/MlWkSJFs55JbOR0z6crVgmtXk5OuXIm49gqZl5eXOnTooA4dOigtLU1t2rTRyJEjFRsbm+3y5jdyC9qNunoOlCxZ8qaO2/79+/XII4/Y3587d04JCQlq3rx5vvZzvTH8/R8iDhw4oIyMjOsuGBIQECAfHx+lp6dfd2xhYWHauXOnjDEOn1NWPx/XKlu2rH7++WedOXMmx6tfuf38//6z+ffzLi0tTYcOHbplxxnA7YlnvgAgH7i7uysqKsrhVbRoUaWnpyspKcmhbsmSJRUSEmJfFrtmzZoqU6aMxo8fnylMXP3X8ODgYFWvXl3Tp093qLNz504tWrTI/st1Ttq0aSNnZ2eNGDEi07+yG2N0+vTp67YRFRWlYsWKafbs2Zo9e7YiIiIcftEuWbKkGjVqpP/+979KSEjItP+1t2ZmJTfHTLryS/TatWuVlpZmL/vhhx8yLZt/7bxcXV1VuXJlGWN06dKlbMdxNcxmFfDyW3R0tHx9fTVq1Kgsx5Sb4yZJH330kcP+kydP1uXLl+2BNb/6ycnEiRMd3k+YMEGSrnuV19nZWW3bttXcuXOz/C69v4+tefPm+uOPP/T111/by86fP5/t7Yp/17ZtWxljsrzq+fefCy8vr1x99lFRUXJ1ddX777/vsP+UKVOUlJSkxx9//LptALh7cOULAK5j+/bt9u+oOnDggJKSkvTmm29KunJ1q0WLFtnu+9dff+mee+5Ru3btVK1aNXl7e2vJkiXasGGD/vOf/0iSnJycNHnyZLVo0ULVq1dXt27dFBwcrL1792rXrl36+eefJUnvvPOOmjVrpjp16qh79+72peb9/Pz02muvXXceZcuW1ZtvvqnY2FgdPnxYrVu3lo+Pjw4dOqRvvvlGvXr10osvvphjG0WKFFGbNm305ZdfKiUlRWPHjs1UZ+LEiapfv76qVKminj176t5771ViYqLWrFmjY8eOadu2bTn2kZtjJkk9evTQ119/raZNm6p9+/aKj4/X559/nmlJ8CZNmigoKEj16tVTYGCg9uzZow8++ECPP/64fHx8cjxe/v7++vDDD+Xj4yMvLy9FRkZm+YzazfL19dXkyZP1zDPPqGbNmnryyScVEBCgI0eO6Mcff1S9evX0wQcfXLedtLQ0NW7cWO3bt1dcXJwmTZqk+vXrq2XLlvnaT04OHTqkli1bqmnTplqzZo0+//xzderUyeFKcHbeeustLV++XJGRkerZs6cqV66sM2fOaPPmzVqyZIn9u9F69uypDz74QJ07d9amTZsUHBysGTNm5Gop/EceeUTPPPOM3n//fe3fv19NmzZVRkaGVq1apUceeUT9+vWTJNWqVUtLlizRuHHjFBISojJlytgXUPm7gIAAxcbGasSIEWratKlatmxpP/YPPvigw+IaAMBS8wDuWleXjN6wYUOu6mX1ymop6r9LTU01L730kqlWrZrx8fExXl5eplq1ambSpEmZ6v7yyy/mscces9erWrWqmTBhgkOdJUuWmHr16hkPDw/j6+trWrRoYXbv3u1Q5+qS3ydPnsxyTHPnzjX169c3Xl5exsvLy1SsWNH07dvXxMXF5TiXqxYvXmwkGZvNZo4ePZplnfj4eNO5c2cTFBRkihQpYkqVKmX+8Y9/mK+//tpeJ7vjn5dj9p///MeUKlXKuLm5mXr16pmNGzdmWmr+v//9r2nQoIEpXry4cXNzM2XLljUvvfSSSUpKuu5c58+fbypXrmxcXFwclp3Pbqn5d955x2H/5cuXG0lmzpw5DuXZzX358uUmOjra+Pn5GXd3d1O2bFnTtWtXs3HjxhzHebW9lStXml69epmiRYsab29v89RTTzl8NUFe+unSpYvx8vK67jG66up5t3v3btOuXTvj4+NjihYtavr162cuXLjgUFdStl8lkJiYaPr27WtCQ0NNkSJFTFBQkGncuLH56KOPHOr99ttvpmXLlsbT09OUKFHCDBgwwCxcuPC6S80bc+XrHd555x1TsWJF4+rqagICAkyzZs3Mpk2b7HX27t1rGjRoYDw8PBx+1q9dav6qDz74wFSsWNEUKVLEBAYGmj59+mT66oiGDRtm+bUHWY0RwJ3JZgxPeAIAcDubNm2aunXrpg0bNtz0yp036uoXDZ88eTJfv6QcAO4kPPMFAAAAABYgfAEAAACABQhfAAAAAGABnvkCAAAAAAtw5QsAAAAALED4AgAAAAAL8CXLNygjI0N//PGHfHx8ZLPZCno4AAAAAAqIMUZ//fWXQkJC5OSU/fUtwtcN+uOPPxQaGlrQwwAAAABQSBw9elT33HNPttsJXzfIx8dH0pUD7OvrW8CjAQAAAFBQkpOTFRoaas8I2SF83aCrtxr6+voSvgAAAABc93EkFtwAAAAAAAsQvgAAAADAAoQvAAAAALAAz3zdQsYYXb58Wenp6QU9FOCuVKRIETk7Oxf0MAAAACQRvm6ZtLQ0JSQk6Pz58wU9FOCuZbPZdM8998jb27ughwIAAED4uhUyMjJ06NAhOTs7KyQkRK6urnwRM2AxY4xOnjypY8eOqXz58lwBAwAABY7wdQukpaUpIyNDoaGh8vT0LOjhAHetgIAAHT58WJcuXSJ8AQCAAseCG7eQkxOHFyhIXHEGAACFCekAAAAAACxA+AIAAAAACxC+4KBRo0YaOHBggfXftWtXtW7dutCMBwAAAMgvLLiBQm3evHkqUqRIQQ8DAAAAuGmEr0IsPcNo/aEzOvHXRZX0cVdEmWJydrq7FhAoVqxYQQ8BAAAAyBfcdlhILdyZoPpvL1PHj9dqwJdb1fHjtar/9jIt3Jlwy/u+fPmy+vXrJz8/P5UoUUJDhw6VMUaSNGPGDNWuXVs+Pj4KCgpSp06ddOLECfu+f/75p5566ikFBATIw8ND5cuX19SpU+3bjx49qvbt28vf31/FihVTq1atdPjw4WzHcu1th+Hh4Ro1apSeffZZ+fj4qHTp0vroo48c9slrHwAAAIAVCF+F0MKdCerz+WYlJF10KD+edFF9Pt98ywPY9OnT5eLiovXr1+u9997TuHHj9Mknn0iSLl26pDfeeEPbtm3Tt99+q8OHD6tr1672fYcOHardu3frp59+0p49ezR58mSVKFHCvm90dLR8fHy0atUqrV69Wt7e3mratKnS0tJyPb7//Oc/ql27trZs2aLnnntOffr0UVxcXL72AQAAAOQ3bjssZNIzjEZ8v1smi21Gkk3SiO9367HKQbfsFsTQ0FC9++67stlsqlChgnbs2KF3331XPXv21LPPPmuvd++99+r999/Xgw8+qHPnzsnb21tHjhxRjRo1VLt2bUlXrlRdNXv2bGVkZOiTTz6xf//S1KlT5e/vrxUrVqhJkya5Gl/z5s313HPPSZIGDx6sd999V8uXL1eFChXyrQ8AAAAgv3Hlq5BZf+hMpitef2ckJSRd1PpDZ27ZGB566CGHL6etU6eO9u/fr/T0dG3atEktWrRQ6dKl5ePjo4YNG0qSjhw5Iknq06ePvvzyS1WvXl0vv/yyfv31V3s727Zt04EDB+Tj4yNvb295e3urWLFiunjxouLj43M9vqpVq9r/brPZFBQUZL/1Mb/6AAAAQCGWkS4dWiXt+PrKnxnpBT2iXOHKVyFz4q/sg9eN1MtPFy9eVHR0tKKjozVz5kwFBAToyJEjio6Ott/S16xZM/32229asGCBFi9erMaNG6tv374aO3aszp07p1q1amnmzJmZ2g4ICMj1OK5d/dBmsykjI0OS8q0PAAAAFFK7v5MWDpaS//i/Mt8QqenbUuWWBTeuXCB8FTIlfdzztd6NWLduncP7tWvXqnz58tq7d69Onz6tt956S6GhoZKkjRs3Zto/ICBAXbp0UZcuXfTwww/rpZde0tixY1WzZk3Nnj1bJUuWlK+v7y0ZuxV9AAAAoIDs/k76qrN07UM6yQlXytt/VqgDGLcdFjIRZYop2M9d2T3NZZMU7Hdl2flb5ciRI4qJiVFcXJy++OILTZgwQQMGDFDp0qXl6uqqCRMm6ODBg/ruu+/0xhtvOOw7bNgwzZ8/XwcOHNCuXbv0ww8/qFKlSpKkp556SiVKlFCrVq20atUqHTp0SCtWrFD//v117NixfBm7FX0AAACgAGSkX7nile3qCJIWDinUtyASvgoZZyebhreoLEmZAtjV98NbVL6l3/fVuXNnXbhwQREREerbt68GDBigXr16KSAgQNOmTdOcOXNUuXJlvfXWWxo7dqzDvq6uroqNjVXVqlXVoEEDOTs768svv5QkeXp66n//+59Kly6tNm3aqFKlSurevbsuXryYb1eprOgDAAAABeC3Xx1vNczESMm/X6lXSNnM1S9wQp4kJyfLz89PSUlJmX6pv3jxog4dOqQyZcrI3f3Gbg9cuDNBI77f7bD4RrCfu4a3qKymDwTf1NiBu0V+/CwCAIBCYsfX0tzu16/XdopUpd2tH8/f5JQN/o5nvgqppg8E67HKQVp/6IxO/HVRJX2u3Gp4K694AQAAAIWWd2D+1isAhK9CzNnJpjplixf0MAAAAICCF1b3yqqGyQnK+rkv25XtYXWtHlmu8cwXAAAAgMLPyfnKcvKSsl0doelbV+oVUoQvAAAAALeHyi2vLCfve80aCL4hhX6ZeYnbDgEAAADcTiq3lCo+fmVVw3OJV57xCqtbqK94XUX4AgAAAHB7cXKWyjxc0KPIM247BAAAAAALEL4AAAAAwAKELwAAAACwAOEL17V69WpVqVJFRYoUUevWrXOsu2LFCtlsNp09e/am+mzUqJEGDhx4U23g9pFf5w0AAEBhRviCg6xCT0xMjKpXr65Dhw5p2rRpBTKuwua1115T9erVC3oYAAAAuI0UePiaOHGiwsPD5e7ursjISK1fvz7H+uPHj1eFChXk4eGh0NBQDRo0SBcvXrRv/9///qcWLVooJCRENptN3377baY2jDEaNmyYgoOD5eHhoaioKO3fvz+/p3bzMtKlQ6ukHV9f+TMjvUCGER8fr0cffVT33HOP/P39C2QMwPWkpaUV9BAAAAByVKDha/bs2YqJidHw4cO1efNmVatWTdHR0Tpx4kSW9WfNmqUhQ4Zo+PDh2rNnj6ZMmaLZs2frlVdesddJSUlRtWrVNHHixGz7HTNmjN5//319+OGHWrdunby8vBQdHe0Q4grc7u+k8Q9I0/8hze1+5c/xD1wpv0W6du2qlStX6r333pPNZrO/Tp8+rWeffVY2my3XV742bdqk2rVry9PTU3Xr1lVcXJxDP9fevjhw4EA1atTIoezy5cvq16+f/Pz8VKJECQ0dOlTGmFz1P2nSJJUvX17u7u4KDAxUu3bt7NsyMjI0evRolSlTRh4eHqpWrZq+/vpr+/art8AtXbo0yzlMmzZNI0aM0LZt2+zH6OpxOXv2rHr06KGAgAD5+vrq0Ucf1bZt2+xtX71iNmPGDIWHh8vPz09PPvmk/vrrL4fxjRkzRuXKlZObm5tKly6tkSNH2rcfPXpU7du3l7+/v4oVK6ZWrVrp8OHDuToukvTJJ5+oUqVKcnd3V8WKFTVp0iT7tsOHD8tms2nevHl65JFH5OnpqWrVqmnNmjUObaxevVqNGjWSp6enihYtqujoaP3555+SpNTUVPXv318lS5aUu7u76tevrw0bNjjsv2DBAt13333y8PDQI488kuX4f/nlFz388MP2f2jp37+/UlJS7NvDw8P1xhtvqHPnzvL19VWvXr1yfQwAAAAKhClAERERpm/fvvb36enpJiQkxIwePTrL+n379jWPPvqoQ1lMTIypV69elvUlmW+++cahLCMjwwQFBZl33nnHXnb27Fnj5uZmvvjii1yPPSkpyUgySUlJmbZduHDB7N6921y4cCHX7TnYNd+Y4X7GDPe95uV35bVr/o21ex1nz541derUMT179jQJCQnm2LFj5tixY8bX19eMHz/eJCQkmPPnz+fYxvLly40kExkZaVasWGF27dplHn74YVO3bl17nS5duphWrVo57DdgwADTsGFD+/uGDRsab29vM2DAALN3717z+eefG09PT/PRRx9ddx4bNmwwzs7OZtasWebw4cNm8+bN5r333rNvf/PNN03FihXNwoULTXx8vJk6dapxc3MzK1asyNUczp8/b1544QVz//33m4SEBIfjEhUVZVq0aGE2bNhg9u3bZ1544QVTvHhxc/r0aWOMMcOHDzfe3t6mTZs2ZseOHeZ///ufCQoKMq+88op9fC+//LIpWrSomTZtmjlw4IBZtWqV+fjjj40xxqSlpZlKlSqZZ5991mzfvt3s3r3bdOrUyVSoUMGkpqZe99h8/vnnJjg42MydO9ccPHjQzJ071xQrVsxMmzbNGGPMoUOHjCRTsWJF88MPP5i4uDjTrl07ExYWZi5dumSMMWbLli3Gzc3N9OnTx2zdutXs3LnTTJgwwZw8edIYY0z//v1NSEiIWbBggdm1a5fp0qWLKVq0qP0YHDlyxLi5uZmYmBj7ZxsYGGgkmT///NMYY8yBAweMl5eXeffdd82+ffvM6tWrTY0aNUzXrl3tcwkLCzO+vr5m7Nix5sCBA+bAgQOZ5nvTP4sAAAC5kFM2+LsCC1+pqanG2dk5Uzjq3LmzadmyZZb7zJw50/j5+Zl169YZY4yJj483FStWNCNHjsyyflbhKz4+3kgyW7ZscShv0KCB6d+/f7bjvXjxoklKSrK/jh49emvCV/plY/5TMYvg9bcA9p9KV+rdAg0bNjQDBgxwKPPz8zNTp07N1f5Xg8uSJUvsZT/++KORZD8euQ1flSpVMhkZGfaywYMHm0qVKl13DHPnzjW+vr4mOTk507aLFy8aT09P8+uvvzqUd+/e3XTs2DHXcxg+fLipVq2aQxurVq0yvr6+5uLFiw7lZcuWNf/973/t+3l6ejqM7aWXXjKRkZHGGGOSk5ONm5ubPWxda8aMGaZChQoOxyU1NdV4eHiYn3/+OcfjcnUss2bNcih74403TJ06dYwx/xe+PvnkE/v2Xbt2GUlmz549xhhjOnbsmO0/eJw7d84UKVLEzJw5016WlpZmQkJCzJgxY4wxxsTGxprKlSs77Dd48GCH8NW9e3fTq1cvhzqrVq0yTk5O9s8gLCzMtG7dOsf5Er4AAIAVchu+XKy+0nbVqVOnlJ6ersDAQIfywMBA7d27N8t9OnXqpFOnTql+/foyxujy5cvq3bu3w22H13P8+HF7P9f2e3VbVkaPHq0RI0bkup8b9tuvUvIfOVQwUvLvV+oV4m/1rlq1qv3vwcHBkqQTJ06odOnSuW7joYceks1ms7+vU6eO/vOf/yg9PV3Ozs7Z7vfYY48pLCxM9957r5o2baqmTZvqn//8pzw9PXXgwAGdP39ejz32mMM+aWlpqlGjxk3NYdu2bTp37pyKFy/uUH7hwgXFx8fb34eHh8vHx8eh7au32u7Zs0epqalq3Lhxtn0cOHDAYX9JunjxokMfWUlJSVF8fLy6d++unj172ssvX74sPz8/h7rZzb1ixYraunWrnnjiiSz7iI+P16VLl1SvXj17WZEiRRQREaE9e/bY5xgZGemwX506dTLNc/v27Zo5c6a9zBijjIwMHTp0SJUqVZIk1a5dO8c5AwAAFCYFFr5uxIoVKzRq1ChNmjRJkZGROnDggAYMGKA33nhDQ4cOvaV9x8bGKiYmxv4+OTlZoaGh+d/RucT8rVdAihQpYv/71QCVkZEhSXJycsr07NalS5fyrW8fHx9t3rxZK1as0KJFizRs2DC99tpr2rBhg86dOydJ+vHHH1WqVCmH/dzc3HI9h6ycO3dOwcHBWrFiRaZtf1+o5O/tXm37arseHh45zu3cuXOqVauWQyi5KiAg4Lr7StLHH3+cKfxcG2Zzmvv1xpgfzp07p3/961/q379/pm1/D79eXl63fCwAAAD5pcDCV4kSJeTs7KzERMcQkZiYqKCgoCz3GTp0qJ555hn16NFDklSlShWlpKSoV69e+ve//y0np+uvH3K17cTERPu/6F99n9PS4W5ubpl+Ob8lvAOvXycv9fLI1dVV6em3dlXFgIAA7dy506Fs69atmULJunXrHN6vXbtW5cuXz/Gq11UuLi6KiopSVFSUhg8fLn9/fy1btkyPPfaY3NzcdOTIETVs2PCG55DVcapZs6aOHz8uFxcXhYeH31C75cuXl4eHh5YuXWo/z6/tY/bs2SpZsqR8fX3z1HZgYKBCQkJ08OBBPfXUUzc0PunKVbGlS5dmeSW4bNmycnV11erVqxUWFibpSrDesGGD/SsMKlWqpO++c1w4Zu3atQ7va9asqd27d6tcuXI3PE4AAIDCpsBWO3R1dVWtWrW0dOlSe1lGRoaWLl2a6Rakq86fP58pYF39RfzaKynZKVOmjIKCghz6TU5O1rp167Lt11JhdSXfEEm2bCrYJN9SV+rdAuHh4Vq3bp0OHz6sU6dO5Xil50Y9+uij2rhxoz777DPt379fw4cPzxTGJOnIkSOKiYlRXFycvvjiC02YMEEDBgy4bvs//PCD3n//fW3dulW//fabPvvsM2VkZKhChQry8fHRiy++qEGDBmn69OmKj4/X5s2bNWHCBE2fPj3XcwgPD9ehQ4e0detWnTp1SqmpqYqKilKdOnXUunVrLVq0SIcPH9avv/6qf//739q4cWOu2nV3d9fgwYP18ssv67PPPlN8fLzWrl2rKVOmSJKeeuoplShRQq1atdKqVat06NAhrVixQv3799exY8eu2/6IESM0evRovf/++9q3b5927NihqVOnaty4cbmee2xsrDZs2KDnnntO27dv1969ezV58mSdOnVKXl5e6tOnj1566SUtXLhQu3fvVs+ePXX+/Hl1795dktS7d2/t379fL730kuLi4jRr1qxMq2gOHjxYv/76q/r166etW7dq//79mj9/vvr165frcQIAABQ2BbrUfExMjD7++GNNnz5de/bsUZ8+fZSSkqJu3bpJkjp37qzY2Fh7/RYtWmjy5Mn68ssvdejQIS1evFhDhw5VixYt7CHs3Llz2rp1q7Zu3SpJ9l+Qjxw5IunKLVQDBw7Um2++qe+++047duxQ586dFRISkmn58wLh5Cw1ffv/v7k2gP3/903fulLvFnjxxRfl7OysypUrKyAgwH7c8lN0dLSGDh2ql19+WQ8++KD++usvde7cOVO9zp0768KFC4qIiFDfvn01YMCAXC0n7u/vr3nz5unRRx9VpUqV9OGHH+qLL77Q/fffL0n221RHjx6tSpUqqWnTpvrxxx9VpkyZXM+hbdu2atq0qR555BEFBAToiy++kM1m04IFC9SgQQN169ZN9913n5588kn99ttvmZ4xzMnQoUP1wgsvaNiwYapUqZI6dOhgfybM09NT//vf/1S6dGm1adNGlSpVUvfu3XXx4sVcXQnr0aOHPvnkE02dOlVVqlRRw4YNNW3atDzN/b777tOiRYu0bds2RUREqE6dOpo/f75cXK5cSH/rrbfUtm1bPfPMM6pZs6YOHDign3/+WUWLFpV05bbBuXPn6ttvv1W1atX04YcfatSoUQ59VK1aVStXrtS+ffv08MMPq0aNGho2bJhCQkJyPU4AAIDCxmZye8noFvnggw/0zjvv6Pjx46pevbref/99+/MojRo1Unh4uP1fxS9fvqyRI0dqxowZ+v333xUQEKAWLVpo5MiR9mdqVqxYoUceeSRTP126dLG3Y4zR8OHD9dFHH+ns2bOqX7++Jk2apPvuuy/X405OTpafn5+SkpIy/dJ78eJFHTp0SGXKlJG7u3veD4p05fu8Fg52XHzDt9SV4FW55Y21Cdxl8uVnEQAA4DpyygZ/V+Dh63Z1y8OXJGWkX1nV8FzilWe8wuresitewJ2I8AUAAKyQ2/BVoLcd4jqcnK8sJ1+l3ZU/C0Hw6t27t7y9vbN89e7d25IxrFq1KtsxeHt7WzKGwiqn47Jq1aqCHh4AAMBd7bZaah4F7/XXX9eLL76Y5ba8rr53o2rXrm1/pg+Ocjou1y6tDwAAAGsRvpAnJUuWVMmSJQt0DB4eHixBng2OCwAAQOHFbYe3EI/TAQWLn0EAAFCYEL5ugatfFnz+/PkCHglwd0tLS5OkXH0xNwAAwK3GbYe3gLOzs/z9/R2+m8lmy+5LkwHcChkZGTp58qQ8PT3t30EGAABQkPiN5BYJCgqSJHsAA2A9JycnlS5dmn/8AAAAhQLh6xax2WwKDg5WyZIldenSpYIeDnBXcnV1lZMTd1cDAIDCgfB1izk7O/O8CQAAAAAW3AAAAAAAKxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALFAowtfEiRMVHh4ud3d3RUZGav369TnWHz9+vCpUqCAPDw+FhoZq0KBBunjxYp7abNSokWw2m8Ord+/e+T43AAAAAJAKQfiaPXu2YmJiNHz4cG3evFnVqlVTdHS0Tpw4kWX9WbNmaciQIRo+fLj27NmjKVOmaPbs2XrllVfy3GbPnj2VkJBgf40ZM+aWzhUAAADA3avAw9e4cePUs2dPdevWTZUrV9aHH34oT09Pffrpp1nW//XXX1WvXj116tRJ4eHhatKkiTp27OhwZSu3bXp6eiooKMj+8vX1vaVzBQAAAHD3KtDwlZaWpk2bNikqKspe5uTkpKioKK1ZsybLferWratNmzbZw9bBgwe1YMECNW/ePM9tzpw5UyVKlNADDzyg2NhYnT9/PtuxpqamKjk52eEFAAAAALnlUpCdnzp1Sunp6QoMDHQoDwwM1N69e7Pcp1OnTjp16pTq168vY4wuX76s3r172287zG2bnTp1UlhYmEJCQrR9+3YNHjxYcXFxmjdvXpb9jh49WiNGjLiZ6QIAAAC4ixVo+LoRK1as0KhRozRp0iRFRkbqwIEDGjBggN544w0NHTo01+306tXL/vcqVaooODhYjRs3Vnx8vMqWLZupfmxsrGJiYuzvk5OTFRoaenOTAQAAAHDXKNDwVaJECTk7OysxMdGhPDExUUFBQVnuM3ToUD3zzDPq0aOHpCvBKSUlRb169dK///3vG2pTkiIjIyVJBw4cyDJ8ubm5yc3NLU/zAwAAAICrCvSZL1dXV9WqVUtLly61l2VkZGjp0qWqU6dOlvucP39eTk6Ow3Z2dpYkGWNuqE1J2rp1qyQpODj4RqcDAAAAANkq8NsOY2Ji1KVLF9WuXVsREREaP368UlJS1K1bN0lS586dVapUKY0ePVqS1KJFC40bN041atSw33Y4dOhQtWjRwh7CrtdmfHy8Zs2apebNm6t48eLavn27Bg0apAYNGqhq1aoFcyAAAAAA3NEKPHx16NBBJ0+e1LBhw3T8+HFVr15dCxcutC+YceTIEYcrXa+++qpsNpteffVV/f777woICFCLFi00cuTIXLfp6uqqJUuW2ENZaGio2rZtq1dffdXayQMAAAC4a9iMMaagB3E7Sk5Olp+fn5KSkvh+MAAAAOAulttsUOBfsgwAAAAAdwPCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGCBQhG+Jk6cqPDwcLm7uysyMlLr16/Psf748eNVoUIFeXh4KDQ0VIMGDdLFixfz1ObFixfVt29fFS9eXN7e3mrbtq0SExPzfW4AAAAAIN1k+Lo28NyI2bNnKyYmRsOHD9fmzZtVrVo1RUdH68SJE1nWnzVrloYMGaLhw4drz549mjJlimbPnq1XXnklT20OGjRI33//vebMmaOVK1fqjz/+UJs2bW56PgAAAACQFZsxxuRlh4yMDI0cOVIffvihEhMTtW/fPt17770aOnSowsPD1b179zwNIDIyUg8++KA++OADe/uhoaF6/vnnNWTIkEz1+/Xrpz179mjp0qX2shdeeEHr1q3TL7/8kqs2k5KSFBAQoFmzZqldu3aSpL1796pSpUpas2aNHnrooUz9pqamKjU11f4+OTlZoaGhSkpKkq+vb57mDAAAAODOkZycLD8/v+tmgzxf+XrzzTc1bdo0jRkzRq6urvbyBx54QJ988kme2kpLS9OmTZsUFRX1fwNyclJUVJTWrFmT5T5169bVpk2b7LcRHjx4UAsWLFDz5s1z3eamTZt06dIlhzoVK1ZU6dKls+139OjR8vPzs79CQ0PzNFcAAAAAd7c8h6/PPvtMH330kZ566ik5Ozvby6tVq6a9e/fmqa1Tp04pPT1dgYGBDuWBgYE6fvx4lvt06tRJr7/+uurXr68iRYqobNmyatSokf22w9y0efz4cbm6usrf3z/X/cbGxiopKcn+Onr0aJ7mCgAAAODulufw9fvvv6tcuXKZyjMyMnTp0qV8GVROVqxYoVGjRmnSpEnavHmz5s2bpx9//FFvvPHGLe3Xzc1Nvr6+Di8AAAAAyC2XvO5QuXJlrVq1SmFhYQ7lX3/9tWrUqJGntkqUKCFnZ+dMqwwmJiYqKCgoy32GDh2qZ555Rj169JAkValSRSkpKerVq5f+/e9/56rNoKAgpaWl6ezZsw5Xv3LqFwAAAABuRp6vfA0bNkz9+vXT22+/rYyMDM2bN089e/bUyJEjNWzYsDy15erqqlq1ajksnpGRkaGlS5eqTp06We5z/vx5OTk5Dvvq7Y/GmFy1WatWLRUpUsShTlxcnI4cOZJtvwAAAABwM/J85atVq1b6/vvv9frrr8vLy0vDhg1TzZo19f333+uxxx7L8wBiYmLUpUsX1a5dWxERERo/frxSUlLUrVs3SVLnzp1VqlQpjR49WpLUokULjRs3TjVq1FBkZKQOHDigoUOHqkWLFvYQdr02/fz81L17d8XExKhYsWLy9fXV888/rzp16mS50iEAAAAA3Kw8ha/Lly9r1KhRevbZZ7V48eJ8GUCHDh108uRJDRs2TMePH1f16tW1cOFC+4IZR44ccbjS9eqrr8pms+nVV1/V77//roCAALVo0UIjR47MdZuS9O6778rJyUlt27ZVamqqoqOjNWnSpHyZEwAAAABcK8/f8+Xt7a2dO3cqPDz8Fg3p9pDbtfwBAAAA3Nlu2fd8NW7cWCtXrrypwQEAAADA3SbPz3w1a9ZMQ4YM0Y4dO1SrVi15eXk5bG/ZsmW+DQ4AAAAA7hR5vu3w2pUGHRqz2ZSenn7Tg7odcNshAAAAACn32SDPV74yMjJuamAAAAAAcDfK8zNfAAAAAIC8u6HwtXLlSrVo0ULlypVTuXLl1LJlS61atSq/xwYAAAAAd4w8h6/PP/9cUVFR8vT0VP/+/dW/f395eHiocePGmjVr1q0YIwAAAADc9vK84EalSpXUq1cvDRo0yKF83Lhx+vjjj7Vnz558HWBhxYIbAAAAAKRb+D1fBw8eVIsWLTKVt2zZUocOHcprcwAAAABwV8hz+AoNDdXSpUszlS9ZskShoaH5MigAAAAAuNPkean5F154Qf3799fWrVtVt25dSdLq1as1bdo0vffee/k+QAAAAAC4E+Q5fPXp00dBQUH6z3/+o6+++krSlefAZs+erVatWuX7AAEAAADgTpDnBTdwBQtuAAAAAJBu4YIbGzZs0Lp16zKVr1u3Ths3bsxrcwAAAABwV8hz+Orbt6+OHj2aqfz3339X375982VQAAAAAHCnyXP42r17t2rWrJmpvEaNGtq9e3e+DAoAAAAA7jR5Dl9ubm5KTEzMVJ6QkCAXlzyv3wEAAAAAd4U8h68mTZooNjZWSUlJ9rKzZ8/qlVde0WOPPZavgwMAAACAO0WeL1WNHTtWDRo0UFhYmGrUqCFJ2rp1qwIDAzVjxox8HyAAAAAA3AnyHL5KlSql7du3a+bMmdq2bZs8PDzUrVs3dezYUUWKFLkVYwQAAACA294NPaTl5eWlXr165fdYAAAAAOCOlednvqZPn64ff/zR/v7ll1+Wv7+/6tatq99++y1fBwcAAAAAd4o8h69Ro0bJw8NDkrRmzRp98MEHGjNmjEqUKKFBgwbl+wABAAAA4E6Q59sOjx49qnLlykmSvv32W7Vr1069evVSvXr11KhRo/weHwAAAADcEfJ85cvb21unT5+WJC1atMi+vLy7u7suXLiQv6MDAAAAgDtEnq98PfbYY+rRo4dq1Kihffv2qXnz5pKkXbt2KTw8PL/HBwAAAAB3hDxf+Zo4caLq1KmjkydPau7cuSpevLgkadOmTerYsWO+DxAAAAAA7gQ2Y4wp6EHcjpKTk+Xn56ekpCT5+voW9HAAAAAAFJDcZoM8X/kCAAAAAOQd4QsAAAAALED4AgAAAAALEL4AAAAAwAI3Fb7eeustnT17Np+GAgAAAAB3rpsKX6NGjdKZM2fyaywAAAAAcMe6qfDFKvUAAAAAkDs88wUAAAAAFnC5mZ13796tkJCQ/BoLAAAAANyxbip8hYaG5tc4AAAAAOCOxm2HAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAXyLXwdPXpUzz77bH41BwAAAAB3lHwLX2fOnNH06dPzqzkAAAAAuKPkeqn57777LsftBw8evOnBAAAAAMCdKtfhq3Xr1rLZbDLGZFvHZrPly6AAAAAA4E6T69sOg4ODNW/ePGVkZGT52rx5860cJwAAAADc1nIdvmrVqqVNmzZlu/16V8UAAAAA4G6W69sOX3rpJaWkpGS7vVy5clq+fHm+DAoAAAAA7jQ2w+WqG5KcnCw/Pz8lJSXJ19e3oIcDAAAAoIDkNhvk+rbDgwcPclshAAAAANygXIev8uXL6+TJk/b3HTp0UGJi4i0ZFAAAAADcaXIdvq696rVgwYIcnwEDAAAAAPyfXIcvAAAAAMCNy3X4stlsmb5EmS9VBgAAAIDcyfVS88YYde3aVW5ubpKkixcvqnfv3vLy8nKoN2/evPwdIQAAAADcAXIdvrp06eLw/umnn873wQAAAADAnSrX4Wvq1Km3chwAAAAAcEdjwQ0AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALFAowtfEiRMVHh4ud3d3RUZGav369dnWbdSokWw2W6bX448/bq+TmJiorl27KiQkRJ6enmratKn2799/3XZ69+59y+YIAAAA4O5W4OFr9uzZiomJ0fDhw7V582ZVq1ZN0dHROnHiRJb1582bp4SEBPtr586dcnZ21hNPPCFJMsaodevWOnjwoObPn68tW7YoLCxMUVFRSklJcWirZ8+eDm2NGTPmls8XAAAAwN2pwMPXuHHj1LNnT3Xr1k2VK1fWhx9+KE9PT3366adZ1i9WrJiCgoLsr8WLF8vT09Mevvbv36+1a9dq8uTJevDBB1WhQgVNnjxZFy5c0BdffOHQlqenp0Nbvr6+t3y+AAAAAO5OBRq+0tLStGnTJkVFRdnLnJycFBUVpTVr1uSqjSlTpujJJ5+Ul5eXJCk1NVWS5O7u7tCmm5ubfvnlF4d9Z86cqRIlSuiBBx5QbGyszp8/n20/qampSk5OdngBAAAAQG4VaPg6deqU0tPTFRgY6FAeGBio48ePX3f/9evXa+fOnerRo4e9rGLFiipdurRiY2P1559/Ki0tTW+//baOHTumhIQEe71OnTrp888/1/LlyxUbG6sZM2bo6aefzrav0aNHy8/Pz/4KDQ29gRkDAAAAuFu5FPQAbsaUKVNUpUoVRURE2MuKFCmiefPmqXv37ipWrJicnZ0VFRWlZs2ayRhjr9erVy/736tUqaLg4GA1btxY8fHxKlu2bKa+YmNjFRMTY3+fnJxMAAMAAACQawUavkqUKCFnZ2clJiY6lCcmJiooKCjHfVNSUvTll1/q9ddfz7StVq1a2rp1q5KSkpSWlqaAgABFRkaqdu3a2bYXGRkpSTpw4ECW4cvNzU1ubm65mRYAAAAAZFKgtx26urqqVq1aWrp0qb0sIyNDS5cuVZ06dXLcd86cOUpNTc3xVkE/Pz8FBARo//792rhxo1q1apVt3a1bt0qSgoOD8zYJAAAAAMiFAr/tMCYmRl26dFHt2rUVERGh8ePHKyUlRd26dZMkde7cWaVKldLo0aMd9psyZYpat26t4sWLZ2pzzpw5CggIUOnSpbVjxw4NGDBArVu3VpMmTSRJ8fHxmjVrlpo3b67ixYtr+/btGjRokBo0aKCqVave+kkDAAAAuOsUePjq0KGDTp48qWHDhun48eOqXr26Fi5caF+E48iRI3JycrxAFxcXp19++UWLFi3Kss2EhATFxMQoMTFRwcHB6ty5s4YOHWrf7urqqiVLltiDXmhoqNq2batXX3311k0UAAAAwF3NZv6+CgVyLTk5WX5+fkpKSuL7wQAAAIC7WG6zQYF/yTIAAAAA3A0IXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCkX4mjhxosLDw+Xu7q7IyEitX78+27qNGjWSzWbL9Hr88cftdRITE9W1a1eFhITI09NTTZs21f79+x3auXjxovr27avixYvL29tbbdu2VWJi4i2bIwAAAIC7W4GHr9mzZysmJkbDhw/X5s2bVa1aNUVHR+vEiRNZ1p83b54SEhLsr507d8rZ2VlPPPGEJMkYo9atW+vgwYOaP3++tmzZorCwMEVFRSklJcXezqBBg/T9999rzpw5Wrlypf744w+1adPGkjkDAAAAuPvYjDGmIAcQGRmpBx98UB988IEkKSMjQ6GhoXr++ec1ZMiQ6+4/fvx4DRs2TAkJCfLy8tK+fftUoUIF7dy5U/fff7+9zaCgII0aNUo9evRQUlKSAgICNGvWLLVr106StHfvXlWqVElr1qzRQw89dN1+k5OT5efnp6SkJPn6+t7EEQAAAABwO8ttNijQK19paWnatGmToqKi7GVOTk6KiorSmjVrctXGlClT9OSTT8rLy0uSlJqaKklyd3d3aNPNzU2//PKLJGnTpk26dOmSQ78VK1ZU6dKls+03NTVVycnJDi8AAAAAyK0CDV+nTp1Senq6AgMDHcoDAwN1/Pjx6+6/fv167dy5Uz169LCXXQ1RsbGx+vPPP5WWlqa3335bx44dU0JCgiTp+PHjcnV1lb+/f677HT16tPz8/Oyv0NDQPM4WAAAAwN2swJ/5uhlTpkxRlSpVFBERYS8rUqSI5s2bp3379qlYsWLy9PTU8uXL1axZMzk53fh0Y2NjlZSUZH8dPXo0P6YAAAAA4C7hUpCdlyhRQs7OzplWGUxMTFRQUFCO+6akpOjLL7/U66+/nmlbrVq1tHXrViUlJSktLU0BAQGKjIxU7dq1JUlBQUFKS0vT2bNnHa5+5dSvm5ub3Nzc8jhDAAAAALiiQK98ubq6qlatWlq6dKm9LCMjQ0uXLlWdOnVy3HfOnDlKTU3V008/nW0dPz8/BQQEaP/+/dq4caNatWol6Uo4K1KkiEO/cXFxOnLkyHX7BQAAAIAbUaBXviQpJiZGXbp0Ue3atRUREaHx48crJSVF3bp1kyR17txZpUqV0ujRox32mzJlilq3bq3ixYtnanPOnDkKCAhQ6dKltWPHDg0YMECtW7dWkyZNJF0JZd27d1dMTIyKFSsmX19fPf/886pTp06uVjoEAAAAgLwq8PDVoUMHnTx5UsOGDdPx48dVvXp1LVy40L4Ix5EjRzI9qxUXF6dffvlFixYtyrLNhIQExcTEKDExUcHBwercubOGDh3qUOfdd9+Vk5OT2rZtq9TUVEVHR2vSpEm3ZpIAAAAA7noF/j1ftyu+5wsAAACAdJt8zxcAAAAA3C0IXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFiA8AUAAAAAFiB8AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAWIHwBAAAAgAUIXwAAAABgAcIXAAAAAFjApaAHgJuTnmG0/tAZnfjrokr6uCuiTDE5O9kKelgAAAAArkH4uo0t3JmgEd/vVkLSRXtZsJ+7hreorKYPBBfgyAAAAABci9sOb1MLdyaoz+ebHYKXJB1Puqg+n2/Wwp0JBTQyAAAAAFkhfN2G0jOMRny/WyaLbVfLRny/W+kZWdUAAAAAUBAIX7eh9YfOZLri9XdGUkLSRa0/dMa6QQEAAADIEeHrNnTir+yD143UAwAAAHDrEb5uQyV93PO1HgAAAIBbj/B1G4ooU0zBfu7KbkF5m66sehhRppiVwwIAAACQA8LXbcjZyabhLSpLUqYAdvX98BaV+b4vAAAAoBAhfN2mmj4QrMlP11SQn+OthUF+7pr8dE2+5wsAAAAoZPiS5dtY0weC9VjlIK0/dEYn/rqokj5XbjXkihcAAABQ+BC+bnPOTjbVKVu8oIcBAAAA4Dq47RAAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIAAAAACxC+AAAAAMAChC8AAAAAsADhCwAAAAAsQPgCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALFIrwNXHiRIWHh8vd3V2RkZFav359tnUbNWokm82W6fX444/b65w7d079+vXTPffcIw8PD1WuXFkffvjhddvp3bv3LZsjAAAAgLubS0EPYPbs2YqJidGHH36oyMhIjR8/XtHR0YqLi1PJkiUz1Z83b57S0tLs70+fPq1q1arpiSeesJfFxMRo2bJl+vzzzxUeHq5FixbpueeeU0hIiFq2bGmv17NnT73++uv2956enrdolgAAAADudgV+5WvcuHHq2bOnunXrZr9C5enpqU8//TTL+sWKFVNQUJD9tXjxYnl6ejqEr19//VVdunRRo0aNFB4erl69eqlatWqZrqh5eno6tOXr63tL5woAAADg7lWgV77S0tK0adMmxcbG2sucnJwUFRWlNWvW5KqNKVOm6Mknn5SXl5e9rG7duvruu+/07LPPKiQkRCtWrNC+ffv07rvvOuw7c+ZMff755woKClKLFi00dOjQbK9+paamKjU11f4+KSlJkpScnJzr+QIAAAC481zNBMaYHOsVaPg6deqU0tPTFRgY6FAeGBiovXv3Xnf/9evXa+fOnZoyZYpD+YQJE9SrVy/dc889cnFxkZOTkz7++GM1aNDAXqdTp04KCwtTSEiItm/frsGDBysuLk7z5s3Lsq/Ro0drxIgRmcpDQ0NzM1UAAAAAd7i//vpLfn5+2W4v8Ge+bsaUKVNUpUoVRUREOJRPmDBBa9eu1XfffaewsDD973//U9++fRUSEqKoqChJUq9evez1q1SpouDgYDVu3Fjx8fEqW7Zspr5iY2MVExNjf5+RkaEzZ86oePHistlst2iGuJWSk5MVGhqqo0ePcsspcsS5gtzgPEFuca4gNzhPbi/GGP31118KCQnJsV6Bhq8SJUrI2dlZiYmJDuWJiYkKCgrKcd+UlBR9+eWXDgtmSNKFCxf0yiuv6JtvvrGvgFi1alVt3bpVY8eOtYeva0VGRkqSDhw4kGX4cnNzk5ubm0OZv79/jmPE7cHX15f/qCFXOFeQG5wnyC3OFeQG58ntI6crXlcV6IIbrq6uqlWrlpYuXWovy8jI0NKlS1WnTp0c950zZ45SU1P19NNPO5RfunRJly5dkpOT49ScnZ2VkZGRbXtbt26VJAUHB+dxFgAAAABwfQV+22FMTIy6dOmi2rVrKyIiQuPHj1dKSoq6desmSercubNKlSql0aNHO+w3ZcoUtW7dWsWLF3co9/X1VcOGDfXSSy/Jw8NDYWFhWrlypT777DONGzdOkhQfH69Zs2apefPmKl68uLZv365BgwapQYMGqlq1qjUTBwAAAHBXKfDw1aFDB508eVLDhg3T8ePHVb16dS1cuNC+CMeRI0cyXcWKi4vTL7/8okWLFmXZ5pdffqnY2Fg99dRTOnPmjMLCwjRy5Ej7lyi7urpqyZIl9qAXGhqqtm3b6tVXX721k0Wh4ubmpuHDh2e6nRS4FucKcoPzBLnFuYLc4Dy5M9nM9dZDBAAAAADctAL/kmUAAAAAuBsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4wm1j4sSJCg8Pl7u7uyIjI7V+/foc68+ZM0cVK1aUu7u7qlSpogULFjhsN8Zo2LBhCg4OloeHh6KiorR//36HOmfOnNFTTz0lX19f+fv7q3v37jp37px9+4oVK9SqVSsFBwfLy8tL1atX18yZM/Nv0sizwnie/N2BAwfk4+PDl7QXAoX1XDHGaOzYsbrvvvvk5uamUqVKaeTIkfkzadyQwnqu/Pzzz3rooYfk4+OjgIAAtW3bVocPH86XOSPvCuI8GTlypOrWrStPT89s/79y5MgRPf744/L09FTJkiX10ksv6fLlyzc1V9wEA9wGvvzyS+Pq6mo+/fRTs2vXLtOzZ0/j7+9vEhMTs6y/evVq4+zsbMaMGWN2795tXn31VVOkSBGzY8cOe5233nrL+Pn5mW+//dZs27bNtGzZ0pQpU8ZcuHDBXqdp06amWrVqZu3atWbVqlWmXLlypmPHjvbtI0eONK+++qpZvXq1OXDggBk/frxxcnIy33///a07GMhWYT1PrkpLSzO1a9c2zZo1M35+fvk+f+ReYT5Xnn/+eVOhQgUzf/58c/DgQbNx40azaNGiW3MgcF2F9Vw5ePCgcXNzM7GxsebAgQNm06ZNpkGDBqZGjRq37mAgWwV1ngwbNsyMGzfOxMTEZPn/lcuXL5sHHnjAREVFmS1btpgFCxaYEiVKmNjY2Hw/BsgdwhduCxEREaZv37729+np6SYkJMSMHj06y/rt27c3jz/+uENZZGSk+de//mWMMSYjI8MEBQWZd955x7797Nmzxs3NzXzxxRfGGGN2795tJJkNGzbY6/z000/GZrOZ33//PduxNm/e3HTr1i3vk8RNK+znycsvv2yefvppM3XqVMJXASus58ru3buNi4uL2bt3b/5MFDetsJ4rc+bMMS4uLiY9Pd1e57vvvjM2m82kpaXd5KyRVwVxnvxddv9fWbBggXFycjLHjx+3l02ePNn4+vqa1NTUPM0R+YPbDlHopaWladOmTYqKirKXOTk5KSoqSmvWrMlynzVr1jjUl6To6Gh7/UOHDun48eMOdfz8/BQZGWmvs2bNGvn7+6t27dr2OlFRUXJyctK6deuyHW9SUpKKFSuW94niphT282TZsmWaM2eOJk6cePOTxU0pzOfK999/r3vvvVc//PCDypQpo/DwcPXo0UNnzpzJn8kjTwrzuVKrVi05OTlp6tSpSk9PV1JSkmbMmKGoqCgVKVIkfw4AcqWgzpPcWLNmjapUqaLAwECHfpKTk7Vr165ct4P8Q/hCoXfq1Cmlp6c7/IdDkgIDA3X8+PEs9zl+/HiO9a/+eb06JUuWdNju4uKiYsWKZdvvV199pQ0bNqhbt265nB3yS2E+T06fPq2uXbtq2rRp8vX1vcEZIr8U5nPl4MGD+u233zRnzhx99tlnmjZtmjZt2qR27drd4GxxMwrzuVKmTBktWrRIr7zyitzc3OTv769jx47pq6++usHZ4kYV1HmSG9n18/c+YC3CF5BPli9frm7duunjjz/W/fffX9DDQSHSs2dPderUSQ0aNCjooaCQy8jIUGpqqj777DM9/PDDatSokaZMmaLly5crLi6uoIeHQuT48ePq2bOnunTpog0bNmjlypVydXVVu3btZIwp6OEByAbhC4VeiRIl5OzsrMTERIfyxMREBQUFZblPUFBQjvWv/nm9OidOnHDYfvnyZZ05cyZTvytXrlSLFi307rvvqnPnznmcIfJDYT5Pli1bprFjx8rFxUUuLi7q3r27kpKS5OLiok8//fQGZ4wbVZjPleDgYLm4uOi+++6z16lUqZKkKyuWwVqF+VyZOHGi/Pz8NGbMGNWoUUMNGjTQ559/rqVLl+Z4azzyX0GdJ7mRXT9/7wPWInyh0HN1dVWtWrW0dOlSe1lGRoaWLl2qOnXqZLlPnTp1HOpL0uLFi+31y5Qpo6CgIIc6ycnJWrdunb1OnTp1dPbsWW3atMleZ9myZcrIyFBkZKS9bMWKFXr88cf19ttvq1evXjc/YdyQwnyerFmzRlu3brW/Xn/9dfn4+Gjr1q365z//mT8HALlWmM+VevXq6fLly4qPj7fX2bdvnyQpLCzsZqaNG1CYz5Xz58/Lycnx1zhnZ2f7GGGdgjpPcqNOnTrasWOHQ5hfvHixfH19Vbly5Vy3g3xU0Ct+ALnx5ZdfGjc3NzNt2jSze/du06tXL+Pv729fveeZZ54xQ4YMsddfvXq1cXFxMWPHjjV79uwxw4cPz3IJV39/fzN//nyzfft206pVqyyX+q1Ro4ZZt26d+eWXX0z58uUdlvpdtmyZ8fT0NLGxsSYhIcH+On36tAVHBdcqrOfJtVjtsOAV1nMlPT3d1KxZ0zRo0MBs3rzZbNy40URGRprHHnvMgqOCrBTWc2Xp0qXGZrOZESNGmH379plNmzaZ6OhoExYWZs6fP2/BkcHfFdR58ttvv5ktW7aYESNGGG9vb7NlyxazZcsW89dffxlj/m+p+SZNmpitW7eahQsXmoCAAJaaL0CEL9w2JkyYYEqXLm1cXV1NRESEWbt2rX1bw4YNTZcuXRzqf/XVV+a+++4zrq6u5v777zc//vijw/aMjAwzdOhQExgYaNzc3Ezjxo1NXFycQ53Tp0+bjh07Gm9vb+Pr62u6detm/w+aMcZ06dLFSMr0atiwYb7PH7lTGM+TaxG+CofCeq78/vvvpk2bNsbb29sEBgaarl278g86BaywnitffPGFqVGjhvHy8jIBAQGmZcuWZs+ePfk7eeRaQZwn2f0esnz5cnudw4cPm2bNmhkPDw9TokQJ88ILL5hLly7l+/yROzZjeCoTAAAAAG41nvkCAAAAAAsQvgAAAADAAoQvAAAAALAA4QsAAAAALED4AgAAAAALEL4AAAAAwAKELwAAAACwAOELAAAAACxA+AIA3BJdu3ZV69atC3oYdxybzaZvv/1WknT48GHZbDZt3br1htvLjzYAALnjUtADAADcfmw2W47bhw8frvfee0/GGItGdHcKDQ1VQkKCSpQokav6Xbt21dmzZ+3h7UbaAADcOMIXACDPEhIS7H+fPXu2hg0bpri4OHuZt7e3vL29C2Jot8ylS5dUpEiRQtWWs7OzgoKCCrwNAEDucNshACDPgoKC7C8/Pz/ZbDaHMm9v70y3HTZq1EjPP/+8Bg4cqKJFiyowMFAff/yxUlJS1K1bN/n4+KhcuXL66aefHPrauXOnmjVrJm9vbwUGBuqZZ57RqVOnsh3btGnT5O/vr2+//Vbly5eXu7u7oqOjdfToUYd68+fPV82aNeXu7q57771XI0aM0OXLl+3bbTabJk+erJYtW8rLy0sjR47Msr/w8HC98cYb6tixo7y8vFSqVClNnDjRoU52bV1vDPv371eDBg3k7u6uypUra/HixQ7tZnXL4K5du/SPf/xDvr6+8vHx0cMPP6z4+Hi99tprmj59uubPny+bzSabzaYVK1Zk2cbKlSsVEREhNzc3BQcHa8iQIQ7jatSokfr376+XX35ZxYoVU1BQkF577bVsPxMAwBWELwCAZaZPn64SJUpo/fr1ev7559WnTx898cQTqlu3rjZv3qwmTZromWee0fnz5yVJZ8+e1aOPPqoaNWpo48aNWrhwoRITE9W+ffsc+zl//rxGjhypzz77TKtXr9bZs2f15JNP2revWrVKnTt31oABA7R7927997//1bRp0zIFrNdee03//Oc/tWPHDj377LPZ9vfOO++oWrVq2rJli4YMGaIBAwZkCkrXtnW9MWRkZKhNmzZydXXVunXr9OGHH2rw4ME5zvv3339XgwYN5ObmpmXLlmnTpk169tlndfnyZb344otq3769mjZtqoSEBCUkJKhu3bpZttG8eXM9+OCD2rZtmyZPnqwpU6bozTffdKg3ffp0eXl5ad26dRozZoxef/31THMGAFzDAABwE6ZOnWr8/PwylXfp0sW0atXK/r5hw4amfv369veXL182Xl5e5plnnrGXJSQkGElmzZo1xhhj3njjDdOkSROHdo8ePWokmbi4uGzHI8msXbvWXrZnzx4jyaxbt84YY0zjxo3NqFGjHPabMWOGCQ4Otr+XZAYOHHid2RsTFhZmmjZt6lDWoUMH06xZsxzbut4Yfv75Z+Pi4mJ+//13+/affvrJSDLffPONMcaYQ4cOGUlmy5YtxhhjYmNjTZkyZUxaWlqWY732M8mqjVdeecVUqFDBZGRk2OtMnDjReHt7m/T0dGNM5s/SGGMefPBBM3jw4Cz7BQBcwTNfAADLVK1a1f53Z2dnFS9eXFWqVLGXBQYGSpJOnDghSdq2bZuWL1+e5fNj8fHxuu+++7Lsx8XFRQ8++KD9fcWKFeXv7689e/YoIiJC27Zt0+rVqx2udKWnp+vixYs6f/68PD09JUm1a9fO1bzq1KmT6f348eMdyq5t63pj2LNnj0JDQxUSEpJtP9faunWrHn744Zt6nmzPnj2qU6eOw6Iq9erV07lz53Ts2DGVLl1akuNnKUnBwcH2zw0AkDXCFwDAMteGApvN5lB29Rf+jIwMSdK5c+fUokULvf3225naCg4OvuFxnDt3TiNGjFCbNm0ybXN3d7f/3cvL64b7uNa1beV2DHnh4eFxQ/vdiKw+y6ufGwAga4QvAEChVbNmTc2dO1fh4eFyccn9/7IuX76sjRs3KiIiQpIUFxens2fPqlKlSvZ24+LiVK5cuXwZ59q1azO9v9pXdq43hkqVKuno0aNKSEiwB81r+7lW1apVNX369GxXU3R1dVV6enqObVSqVElz586VMcYehlevXi0fHx/dc889Oe4LAMgZC24AAAqtvn376syZM+rYsaM2bNig+Ph4/fzzz+rWrVuOIaJIkSJ6/vnntW7dOm3atEldu3bVQw89ZA9jw4YN02effaYRI0Zo165d2rNnj7788ku9+uqrNzTO1atXa8yYMdq3b58mTpyoOXPmaMCAATnuc70xREVF6b777lOXLl20bds2rVq1Sv/+979zbLNfv35KTk7Wk08+qY0bN2r//v2aMWOG/WsAwsPDtX37dsXFxenUqVO6dOlSpjaee+45HT16VM8//7z27t2r+fPna/jw4YqJiZGTE782AMDN4L+iAIBCKyQkRKtXr1Z6erqaNGmiKlWqaODAgfL3988xCHh6emrw4MHq1KmT6tWrJ29vb82ePdu+PTo6Wj/88IMWLVqkBx98UA899JDeffddhYWF3dA4X3jhBW3cuFE1atTQm2++qXHjxik6OjrHfa43BicnJ33zzTe6cOGCIiIi1KNHj2yXu7+qePHiWrZsmc6dO6eGDRuqVq1a+vjjj+1XwXr27KkKFSqodu3aCggI0OrVqzO1UapUKS1YsEDr169XtWrV1Lt3b3Xv3v2GgykA4P/YjDGmoAcBAEB+mTZtmgYOHKizZ89a0l94eLgGDhyogQMHWtIfAOD2xZUvAAAAALAA4QsAAAAALMBthwAAAABgAa58AQAAAIAFCF8AAAAAYAHCFwAAAABYgPAFAAAAABYgfAEAAACABQhfAAAAAGABwhcAAAAAWIDwBQAAAAAW+H/D1ohoZrTlGwAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "cU4RZomQNwOk"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}