analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy as sp\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "old_m16_path = 'tactics/data/stats/metrics_test_t_m1600_old.csv'\n",
    "none_m16_path = 'tactics/data/stats/metrics_test_t_m1600_none.csv'\n",
    "prec_m16_path = 'tactics/data/stats/metrics_test_t_m1600_prec.csv'\n",
    "rec_m16_path = 'tactics/data/stats/metrics_test_t_m1600_rec.csv'\n",
    "new_m16_path = 'tactics/data/stats/metrics_test_t_m1600_all.csv'\n",
    "\n",
    "old_sf14_path = 'tactics/data/stats/metrics_test_t_sf14_old.csv'\n",
    "none_sf14_path = 'tactics/data/stats/metrics_test_t_sf14_none.csv'\n",
    "prec_sf14_path = 'tactics/data/stats/metrics_test_t_sf14_prec.csv'\n",
    "rec_sf14_path = 'tactics/data/stats/metrics_test_t_sf14_rec.csv'\n",
    "new_sf14_path = 'tactics/data/stats/metrics_test_t_sf14_all.csv'\n",
    "\n",
    "old_m16_raw = pd.read_csv(old_m16_path)\n",
    "none_m16_raw = pd.read_csv(none_m16_path)\n",
    "rec_m16_raw = pd.read_csv(rec_m16_path)\n",
    "new_m16_raw = pd.read_csv(new_m16_path)\n",
    "\n",
    "old_sf14_raw = pd.read_csv(old_sf14_path)\n",
    "none_sf14_raw = pd.read_csv(none_sf14_path)\n",
    "prec_sf14_raw = pd.read_csv(prec_sf14_path)\n",
    "rec_sf14_raw = pd.read_csv(rec_sf14_path)\n",
    "new_sf14_raw = pd.read_csv(new_sf14_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>match</th>\n",
       "      <th>num_suggestions</th>\n",
       "      <th>correct_move</th>\n",
       "      <th>tactic_text</th>\n",
       "      <th>move</th>\n",
       "      <th>position</th>\n",
       "      <th>tactic_ground_div</th>\n",
       "      <th>tactic_ground_avg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>ground</td>\n",
       "      <td>c1b2</td>\n",
       "      <td>r3k2r/bpp1n1pp/p1np4/P3p3/1P6/2P1P3/3P1PPP/RNB...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>random</td>\n",
       "      <td>c1b2</td>\n",
       "      <td>r3k2r/bpp1n1pp/p1np4/P3p3/1P6/2P1P3/3P1PPP/RNB...</td>\n",
       "      <td>21.0</td>\n",
       "      <td>21.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>sf14</td>\n",
       "      <td>c1b2</td>\n",
       "      <td>r3k2r/bpp1n1pp/p1np4/P3p3/1P6/2P1P3/3P1PPP/RNB...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>maia_1600</td>\n",
       "      <td>c1b2</td>\n",
       "      <td>r3k2r/bpp1n1pp/p1np4/P3p3/1P6/2P1P3/3P1PPP/RNB...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>f(A,B,C):-legal_move(B,C,A),attacks(B,D,A),beh...</td>\n",
       "      <td>c1b2</td>\n",
       "      <td>r3k2r/bpp1n1pp/p1np4/P3p3/1P6/2P1P3/3P1PPP/RNB...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   match  num_suggestions  correct_move  \\\n",
       "0      1              1.0           1.0   \n",
       "1      1              1.0           0.0   \n",
       "2      1              1.0           0.0   \n",
       "3      1              1.0           0.0   \n",
       "4      0              NaN           NaN   \n",
       "\n",
       "                                         tactic_text  move  \\\n",
       "0                                             ground  c1b2   \n",
       "1                                             random  c1b2   \n",
       "2                                               sf14  c1b2   \n",
       "3                                          maia_1600  c1b2   \n",
       "4  f(A,B,C):-legal_move(B,C,A),attacks(B,D,A),beh...  c1b2   \n",
       "\n",
       "                                            position  tactic_ground_div  \\\n",
       "0  r3k2r/bpp1n1pp/p1np4/P3p3/1P6/2P1P3/3P1PPP/RNB...                0.0   \n",
       "1  r3k2r/bpp1n1pp/p1np4/P3p3/1P6/2P1P3/3P1PPP/RNB...               21.0   \n",
       "2  r3k2r/bpp1n1pp/p1np4/P3p3/1P6/2P1P3/3P1PPP/RNB...                2.0   \n",
       "3  r3k2r/bpp1n1pp/p1np4/P3p3/1P6/2P1P3/3P1PPP/RNB...                5.0   \n",
       "4  r3k2r/bpp1n1pp/p1np4/P3p3/1P6/2P1P3/3P1PPP/RNB...                NaN   \n",
       "\n",
       "   tactic_ground_avg  \n",
       "0                0.0  \n",
       "1               21.0  \n",
       "2                2.0  \n",
       "3                5.0  \n",
       "4                NaN  "
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "old_m16_raw.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_metrics(df): # TODO: refactor this and the method in analysis.py\n",
    "    agg = df.groupby('tactic_text').aggregate(np.nansum).reset_index()\n",
    "    agg['avg_tactic_ground_avg'] = agg['tactic_ground_avg'] / agg['match']\n",
    "    agg['coverage'] = agg['match'] / df.groupby(['position', 'move']).ngroups\n",
    "    agg['accuracy'] = agg['correct_move'] / agg['match']\n",
    "    return agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_frequency_graph(df, metric_fname: str, title: str='?', xlabel: str=None, bins: int=20, left: int=0, right=None):\n",
    "    mask = df['tactic_text'].isin(['ground', 'sf14', 'maia_1600', 'random'])\n",
    "    df_masked = df[~mask]\n",
    "    plt.hist(df_masked[metric_fname], bins=bins) # exclude default tactics from histogram\n",
    "    plt.axvline(df[df['tactic_text'] == \"random\"][metric_fname].values, linestyle='dashed', color='blue', label='random move tactic')\n",
    "    plt.axvline(df[df['tactic_text'] == \"ground\"][metric_fname].values, linestyle='dashed', color='green', label='ground move tactic')\n",
    "    plt.axvline(df[df['tactic_text'] == \"sf14\"][metric_fname].values, linestyle='dashed', color='red', label='sf14 move tactic')\n",
    "    plt.axvline(df[df['tactic_text'] == \"maia_1600\"][metric_fname].values, linestyle='dashed', color='magenta', label='maia-1600 move tactic')\n",
    "    plt.title(f'Histogram of {title}')\n",
    "    plt.xlabel(xlabel if xlabel else metric_fname)\n",
    "    plt.ylabel('Frequency')\n",
    "    plt.xlim(left=left, right=right)\n",
    "    plt.legend(loc=1)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "old_m16 = calculate_metrics(old_m16_raw)\n",
    "none_m16 = calculate_metrics(none_m16_raw)\n",
    "prec_m16 = calculate_metrics(prec_sf14_raw)\n",
    "rec_m16 = calculate_metrics(rec_m16_raw)\n",
    "new_m16 = calculate_metrics(new_m16_raw)\n",
    "\n",
    "old_sf14 = calculate_metrics(old_sf14_raw)\n",
    "none_sf14 = calculate_metrics(none_sf14_raw)\n",
    "prec_sf14 = calculate_metrics(prec_sf14_raw)\n",
    "rec_sf14 = calculate_metrics(rec_sf14_raw)\n",
    "new_sf14 = calculate_metrics(new_sf14_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_tactic = new_m16[new_m16['tactic_text'] == \"random\"]\n",
    "ground_tactic = new_m16[new_m16['tactic_text'] == \"ground\"]\n",
    "m1600_tactic = new_m16[new_m16['tactic_text'] == \"maia_1600\"]\n",
    "sf14_tactic = new_m16[new_m16['tactic_text'] == \"sf14\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAHHCAYAAABeLEexAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAACByElEQVR4nO3dd1gURx8H8O9x3B0d6YiiYGwo2HsJGmvsGjuxJzH2bjSJsSTBFo2JJiZGIyaKmMTyGguxF6yIolGxo0aFYCEgHe7m/YOw8Whyxymc9/08zz7P7uzM7OwecD9mZ3dkQggBIiIiIhNmVtINICIiIippDIiIiIjI5DEgIiIiIpPHgIiIiIhMHgMiIiIiMnkMiIiIiMjkMSAiIiIik8eAiIiIiEweAyIiIiIyeQyIiIiIyOQxICIiIiKTx4CI8ggKCoJMJsOZM2fy3d+lSxd4eXnlyX/79u0iH+P48eOYM2cO/vnnn+I11oRs2rQJNWvWhKWlJWQyGSIjIwvMm/OZ5CwWFhZwd3dH69atMX/+fMTFxeWbX5fPkHQ3b9481KhRAxqNBgBw+/Ztrc+psOXq1at6HfPZn4VDhw7l2S+EQOXKlSGTydCqVSu96zfkz87Tp08xffp0tG/fHi4uLpDJZJgzZ06hZcLCwtCpUyc4ODjA0tISVapUwaeffqqVJykpCRMnToSHhwcsLCxQp04dhISE5FufLnmLateuXVqfqVwuR8WKFTF27FgkJiYWq+6iWLNmDcqVK4fk5OQXfixjxICIiq1z5844ceIEypYtW+Qyx48fx9y5cxkQFdHDhw8xaNAgvPbaawgNDcWJEydQtWrV55Zbu3YtTpw4gb179+Kbb75BnTp1sHDhQvj4+GDfvn1SPn0+Q9LNgwcPsGjRIsybNw9mZtl/eh0dHXHixAlp2bt3LwCgR48eWulF/bwLY2trizVr1uRJP3z4MG7evAlbW1u96n0RPzuPHz/GqlWrkJ6ejh49ejw3f3BwMPz9/WFvb4+ffvoJu3btwgcffIDcc5f36tUL69atw+zZs7F79240bNgQAwYMQHBwcJ46dclbVGfPngUAbN68GSdOnMD+/fvRpUsXfPPNNxg/frze9RbVkCFDYG1tjUWLFr3wYxklQZTL2rVrBQARHh6e7/7OnTuLihUrFusYixcvFgBEdHR0seopjuTk5BI7tq7CwsIEALFp06Yi5S/sM7xz547w9PQUtra2IjY21tBN1YkxfQbFNX36dFGuXDmhVqsLzJPzOS9fvtxgx835WXjnnXeEpaWlSEhI0Nr/9ttvi6ZNm4qaNWsKf39/gx23ODQajdBoNEIIIR4+fCgAiNmzZ+eb9969e8La2lqMGjWq0Dp37twpAIjg4GCt9Hbt2gkPDw+RlZWlV15d9OzZU1hZWeX5GfDx8RGOjo561amrL774Qtjb25vU715RsYeIii2/LvOHDx/ivffeg6enJ1QqFVxcXNC8eXPs27cPc+bMwbRp0wAA3t7e+Xbnh4WFoU2bNrC1tYWVlRWaNWuGnTt35jn2//73P9SqVQsqlQqVKlXCV199hTlz5kAmk2nly0k7e/YsevfuDQcHB7z22msAgBs3bmDYsGGoUqUKrKysUK5cOXTt2hV//vlnvnVcuHABffr0gb29PRwdHTF58mRkZWXh6tWr6NixI2xtbeHl5VXk/8Ked65Dhw5FixYtAAD9+vXT+9ZGjgoVKmDJkiV4+vQpvv/+ewB5P8Nt27ZBJpNh//79ecqvXLlSug45rl+/joEDB8LV1RUqlQo+Pj745ptvtMoV9hkARf8sdT3epUuXMGDAANjb28PNzQ3Dhw9HQkKCVt4rV65gwIABcHNzg0qlQoUKFTB48GCkp6frfNz8ZGRkYM2aNRg4cKDUO5SfnB6EevXqPbdOXQ0YMAAAsHHjRiktISEBmzdvxvDhw/PkL+rvRX6//0UtW5CcvwlFsXr1aiQnJ+ODDz4oNN/WrVthY2ODPn36aKUPGzYMDx48wKlTp/TKq4uIiAj4+fnl+Rmws7N7abexAgICkJiYWOzbf68iBkRUILVajaysrDyLyNUNnZ9BgwZh27Zt+OSTT7Bnzx6sXr0abdu2xePHj/HOO+9g3LhxAIAtW7ZItwRyvgQOHz6MN954AwkJCVizZg02btwIW1tbdO3aFZs2bZKOERoail69esHJyQmbNm3CokWLsHHjRqxbt67AdvXq1QuVK1fGr7/+iu+++w5A9q0MJycnLFiwAKGhofjmm29gbm6Oxo0b5ztuo2/fvqhduzY2b96Md999F19++SUmTZqEHj16oHPnzti6dSveeOMNfPDBB9iyZUuh16ko5zpr1izpSzcwMBAnTpzAt99++9zPoDCdOnWCXC7HkSNH8t3fpUsXuLq6Yu3atXn2BQUFoV69eqhVqxYA4PLly2jYsCEuXryIJUuWYMeOHejcuTPGjx+PuXPn5imf32egy2ep6/HeeustVK1aFZs3b8aMGTMQHByMSZMmSfvPnz+Phg0b4uTJk5g3bx52796N+fPnIz09HRkZGXof91mnTp3C48eP0bp160LznTt3DmZmZqhdu3ah+fRhZ2eH3r1748cff5TSNm7cCDMzM/Tr1y9Pfl1/LwxVVldHjhyBo6Mjrly5gjp16sDc3Byurq54//33tcblXLx4ET4+PjA3N9cqn/NzfPHiRb3yFtXjx49x9+7dPJ/tw4cPcfHiRTRs2FDnOvXh7u6O6tWr5/sPpskr6S4qKn1yutgLW569ZZaT/9nbXzY2NmLixIkFHqOwW2ZNmjQRrq6u4unTp1JaVlaW8PX1FeXLl5e60hs2bCg8PT1Fenq6lO/p06fCyclJ5P7Rnj17tgAgPvnkk+eef1ZWlsjIyBBVqlQRkyZNylPHkiVLtPLXqVNHABBbtmyR0jIzM4WLi4vo1atXoccq6rkePHhQABC//vrrc9svxPNvewohhJubm/Dx8dHK/+znMXnyZGFpaSn++ecfKe3y5ct5bul06NBBlC9fPs+tmLFjxwoLCwvx5MkTIUThn4Eun6Wux1u0aJFWvtGjRwsLCwvp2r7xxhuiTJkyIi4ursBrpctx87Nw4UIB4Lm3KOvUqSOqV69eaB5dPfuzkPNzdPHiRSFE9nUfOnSoEEI895ZZQb8X+f3sFLVsUTzvllm1atWEhYWFsLW1FYGBgeLgwYNi0aJFwtLSUjRv3lz6nKtUqSI6dOiQp/yDBw8EABEYGCil6ZK3qPbs2SMAiK+//lpkZmaKpKQkcfLkSdGsWTNhbm4uDhw4IITIvl1oa2srYmJidD7GswqrJyAgQLi5uRWr/lcRe4ioQD/99BPCw8PzLDm3bwrTqFEjBAUF4bPPPsPJkyeRmZlZpGMmJyfj1KlT6N27N2xsbKR0uVyOQYMG4d69e7h69SqSk5Nx5swZ9OjRA0qlUspnY2ODrl27Flj/W2+9lSctKysLgYGBqFGjBpRKJczNzaFUKnH9+nVERUXlyd+lSxetbR8fH8hkMrz55ptSmrm5OSpXrow7d+4U+1xfFPGcnr7hw4cjNTVVq1du7dq1UKlUGDhwIAAgLS0N+/fvR8+ePWFlZaXVk9ipUyekpaXh5MmTWvXm/gx0+Sz1OV63bt20tmvVqoW0tDTExcUhJSUFhw8fRt++feHi4lLgtdDnuM968OABZDIZnJ2dC8yTkZGBS5cuvZDbZTn8/f3x2muv4ccff8Sff/6J8PDwfG+XAbr/XuhTVp/e59w0Gg3S0tLw4YcfYubMmWjVqhWmTZuG+fPn49ixY1q3fQu7DZd7ny55iyIiIgIAMH78eCgUCtjY2KBJkybIyMjAH3/8IfUe3r59GyqVCu7u7jof41mF1ePq6oq4uDhkZWUV6xivGgZEVCAfHx80aNAgz2Jvb//csps2bcKQIUOwevVqNG3aFI6Ojhg8eDBiY2MLLRcfHw8hRL5PrHh4eADI7nrOyefm5pYnX35pOfKrd/LkyZg1axZ69OiB33//HadOnUJ4eDhq166N1NTUPPkdHR21tpVKJaysrGBhYZEnPS0trcC2FPVcX4Tk5GQ8fvxYOk5+atasiYYNG0q3zdRqNdavX4/u3btL1+Dx48fIysrC8uXLoVAotJZOnToBAB49eqRVb+7z1eWz1Od4Tk5OWtsqlQoAkJqaivj4eKjVapQvX77A66DvcZ+VmpoKhUIBuVxeYJ6LFy8iMzPzhQZEMpkMw4YNw/r16/Hdd9+hatWqaNmyZb55df290LXs7du381zLw4cP63xOOZ9vhw4dtNJz/kHJGZfl5OSU7+/TkydPAGj/XuuSt6jOnj0LCwsLnD59GuHh4Th//jwePXqE8PBwvPHGGwCyb8v6+PggPj4eNjY2qF+/PoDsoG/p0qWoWrUqHBwcMGTIEOl27pUrV9C2bVs4OjrCwcEB48aNK7CeHBYWFhBCFPr3yRSZPz8Lke6cnZ2xbNkyLFu2DHfv3sX27dsxY8YMxMXFITQ0tMByDg4OMDMzQ0xMTJ59Dx48kOp2cHCATCbD33//nSdfYUFXfv/ZrV+/HoMHD0ZgYKBW+qNHj1CmTJkC6yquop7ri7Bz506o1ernDs4eNmwYRo8ejaioKNy6dQsxMTEYNmyYtN/BwUHq0RozZky+dXh7e2tt5/4MdPks9TleYRwdHSGXy3Hv3r1C8xX3uM7OzsjIyEBycjKsra3zzXPu3DkAL2ZA9bOGDh2KTz75BN999x0+//zzAvMV5/eiKGU9PDwQHh6utb9atWpFP5F/1apVK9/euZzeppwBzH5+fti4cSOysrK0xgblDPT29fWV0nTJW1Rnz55FrVq1Ch0rVKNGDcybNw+3b9/WGif4ySef4OjRozh48CDs7e3RvXt3/PDDDxgzZgwCAgIwZcoU7N27F0+fPsW1a9cKrCfHkydPoFKptHqmiT1E9BJUqFABY8eORbt27aT/1p79L/1Z1tbWaNy4MbZs2aK1T6PRYP369ShfvjyqVq0Ka2trNGjQANu2bdMa+JqUlIQdO3bo1D6ZTCa1J8fOnTtx//59nerRVVHP1dDu3r2LqVOnwt7eHiNHjiw074ABA2BhYYGgoCAEBQWhXLlyaN++vbTfysoKrVu3xrlz51CrVq18exRz99DkpstnaYjjPcvS0hL+/v749ddfC+3hKe5xq1evDgC4efNmgXlyfjfq1q1b5Pbro1y5cpg2bRq6du2KIUOGFJivOL8XRSmrVCrzXEN93oWUcwt29+7dWum7du0CADRp0gQA0LNnTyQlJWHz5s1a+datWwcPDw80btxYStMlb1EkJCTg1q1beXpq8nPhwgWtgdcxMTH46quvsHHjRpQrVw42NjYYMGCAdAvu1q1bUKvV0Gg0sLOzQ4MGDfKt51m3bt1CjRo1dDoHU8AeIjK4hIQEtG7dGgMHDkT16tVha2uL8PBw6UkiIPs/MAD46quvMGTIECgUClSrVg22traYP38+2rVrh9atW2Pq1KlQKpX49ttvcfHiRWzcuFHqYZg3bx46d+6MDh06YMKECVCr1Vi8eDFsbGykru2i6NKlC4KCglC9enXUqlULERERWLx48XNvoxhCUc9VXxcvXpTGZ8TFxeHo0aNYu3Yt5HI5tm7dWui4GQAoU6YMevbsiaCgIPzzzz+YOnVqnkeGv/rqK7Ro0QItW7bEqFGj4OXlhadPn+LGjRv4/fffceDAgee2U5fP0hDHe9bSpUvRokULNG7cGDNmzEDlypXx999/Y/v27fj++++lL+niHDenJ+7kyZPSk0q5nTt3Dt7e3oX2vshkMvj7++f7xmldLFiw4Ll5ivN7YYjfqd27dyM5ORlPnz4FkH076bfffgOQ/ZSklZUVAKB9+/bo2rUr5s2bB41GgyZNmuDMmTOYO3cuunTpIo15fPPNN9GuXTuMGjUKiYmJqFy5MjZu3IjQ0FCsX79e63amLnmB538uZ8+ehRCiSAHR+fPnMXbsWGl73759SEtL0wpg1Go13nnnHQDZTwp+/vnnmDZtGgYNGoTAwEAoFIo89eTQaDQ4ffo0RowY8dy2mJwSG85NpZauL2bM/ZRJWlqaeP/990WtWrWEnZ2dsLS0FNWqVROzZ8/WehnYzJkzhYeHhzAzMxMAxMGDB6V9R48eFW+88YawtrYWlpaWokmTJuL333/P05atW7cKPz8/oVQqRYUKFcSCBQvE+PHjhYODg1a+nCeOHj58mKeO+Ph4MWLECOHq6iqsrKxEixYtxNGjR4W/v7/WUzcF1TFkyBBhbW2dp15/f39Rs2bNfK/hs4pyrvo+ZZazKJVK4erqKvz9/UVgYGCeJ6oKe1Io5+kYAOLatWv5Hi86OloMHz5clCtXTigUCuHi4iKaNWsmPvvsMylPYZ+BEEX/LIt7vPzO9fLly6JPnz7CyclJOv7QoUNFWlqazsctSMuWLUWnTp3y3adWq4W1tbV46623Ciz/9OlTAUD079//ucd6VlGeOBQi71NmRf29yO96FrVsYSpWrFjgU665f05TUlLEBx98IDw9PYW5ubmoUKGCmDlzZp7P7+nTp2L8+PHC3d1dKJVKUatWLbFx48Z8j1/UvEX5XL744gsBQJw9e7bQc05PTxdKpVIkJSVJacuWLZOeBCzM7du3RYUKFcTOnTvzrSfH/v37BQARERHx3DpNDQMieqVkZGSIGjVqiHbt2pV0U6iYXrXP8rfffhNyuVzcu3dPr/I7d+4UMplMXLhwwcAto+Iw5Ofy6NEjoVAotF7hcOTIEVG2bFlx+fJlKc/u3buFEEJs3rxZ3Lp1SwghxLlz54Sbm5u4detWvvXkePvtt0WzZs2K3dZXEccQkVEbMWIEQkJCcPjwYWzatAnt27dHVFQUpk+fXtJNIx296p9lr1690LBhQ8yfP1+v8gcPHkT//v2l281UOhjyc3FycsLAgQNRoUIFNGrUCADQsmVLTJo0Ce3bt4eNjQ0aNWokvSX+8OHDaNq0KWxsbDB48GCsWrUK3t7e+dYDZI9h27RpExYuXFjstr6KZELo8eIHolKib9++OH78OB4+fAiFQoF69erhww8/RMeOHUu6aaQjU/gsL168KD1xWdgUHkQvwsGDB3H9+nW89957Jd2UUokBEREREZk8/otCREREJq9EA6IjR46ga9eu8PDwgEwmw7Zt26R9mZmZ+OCDD+Dn5wdra2t4eHhg8ODB0gvrcqSnp2PcuHFwdnaGtbU1unXrlucla/Hx8Rg0aBDs7e1hb2+PQYMG4Z9//nkJZ0hERETGoEQDouTkZNSuXRsrVqzIsy8lJQVnz57FrFmzcPbsWWzZsgXXrl3LMy/RxIkTsXXrVoSEhCAsLAxJSUno0qUL1Gq1lGfgwIGIjIxEaGgoQkNDERkZiUGDBr3w8yMiIiLjUGrGEMlkMmzduhU9evQoME94eDgaNWqEO3fuoEKFCkhISICLiwt+/vln9OvXD0D2lAeenp7YtWsXOnTogKioKNSoUQMnT56U3i568uRJNG3aFFeuXCnyq+I1Gg0ePHgAW1vbYr8sj4iIiF4OIQSePn0KDw+PQh9mMKo3VSckJEAmk0lvco2IiEBmZqbWVAIeHh7w9fXF8ePH0aFDB5w4cQL29vZar1pv0qQJ7O3tcfz48QIDovT0dKSnp0vb9+/f56vOiYiIjNRff/1V6NvSjSYgSktLw4wZMzBw4EDY2dkByJ74UalUwsHBQSuvm5ubNClkbGwsXF1d89Tn6upa6CSg8+fPx9y5c/Ok//XXX5BbyPHagtcQu+Tf8g8A5D9fY17JyUDODOMPHgAFTPRoDF6hUyEioldUYmIiPD09nztXnlEERJmZmejfvz80Gk2+M/fmJoTQuq2V3y2u3HlymzlzJiZPnixt51xQOzs7yC3kkKlksEN2YAY7FD0genYOHDs7o44iXqFTISKiV9zzhruU+sfuMzMz0bdvX0RHR2Pv3r1S7xAAuLu7IyMjA/Hx8Vpl4uLi4ObmJuX5+++/89T78OFDKU9+VCoV7OzstBYiIiJ6NZXqgCgnGLp+/Tr27dsHJycnrf3169eHQqHA3r17pbSYmBhcvHgRzZo1AwA0bdoUCQkJOH36tJTn1KlTSEhIkPIQERGRaSvRW2ZJSUm4ceOGtB0dHY3IyEg4OjrCw8MDvXv3xtmzZ7Fjxw6o1WppzI+joyOUSiXs7e0xYsQITJkyBU5OTnB0dMTUqVPh5+eHtm3bAgB8fHzQsWNHvPvuu/j+++8BAO+99x66dOlS5CfMcjM3M8eAOgMQ1joMTcs3hdxc/vxCUmFzYMiQ/9aN2Ct0KkREZOJK9LH7Q4cOoXXr1nnShwwZgjlz5sDb2zvfcgcPHkSrVq0AZA+2njZtGoKDg5Gamoo2bdrg22+/haenp5T/yZMnGD9+PLZv3w4A6NatG1asWCE9rVYUiYmJsLe3R0JCAm+fEZFBaTQaZGRklHQziIySQqGAXF5wx0RRv79LzXuISjsGRET0ImRkZCA6Ohoajaakm0JktMqUKQN3d/d8B04X9fubNzr0IIRASkYKkAJYKawgs5YBRX1XoxBASkr2upUVYMQveXyFToWoRAghEBMTA7lcDk9Pz0JfGkdEeQkhkJKSgri4OABA2bJl9a6LAZEeUjJT4DrPFcmBydkJSSj6Y/cpKYCNzb/lkoz6WfVX6FSISkRWVhZSUlLg4eEBKyurkm4OkVGytLQEkP2Euaura6G3zwrDf0eIiEpIzpyLSqWyhFtCZNxy/qHIzMzUuw4GREREJYzzIxIVjyF+hxgQERERkcljQERERKXa0KFD0aNHj5JuBj1DJpNh27ZtJd0Mg2JAREREZOQOHToEmUyGf/75x6D1zpkzB3Xq1MmTHhMTgzfffNOgxyppfMqMiIiKJSMjgwPDTYy7u3tJN8Hg2EOkB7mZHN1rdMeJJiegfksN6PKEn1wO9O6dvej5aGBp8QqdChHpoFWrVhg7diwmT54MZ2dntGvXDgCwdOlS+Pn5wdraGp6enhg9ejSSkpKkckFBQShTpgz++OMP+Pj4wMbGBh07dkRMTIyUR61WY/LkyShTpgycnJwwffp05H5/cHp6OsaPHw9XV1dYWFigRYsWCA8Pl/bn9Jb88ccfqFu3LiwtLfHGG28gLi4Ou3fvho+PD+zs7DBgwACk5LxMLR857d2xYweqVasGKysr9O7dG8nJyVi3bh28vLzg4OCAcePGSU8MAkB8fDwGDx4MBwcHWFlZ4c0338T169cBAAkJCbC0tERoaKjWsbZs2QJra2vpet2/fx/9+vWDg4MDnJyc0L17d9y+fTvfdt6+fVua9cHBwQEymQxDhw4FAISGhqJFixbS9ezSpQtu3rypVf7evXvo378/HB0dYW1tjQYNGuDUqVMICgrC3Llzcf78echkMshkMgQFBQHIe8usoDqMiqAiSUhIEABEQkJCSTeFiF4Rqamp4vLlyyI1NVUrPSmp4CVX1kLzpqQULa+u/P39hY2NjZg2bZq4cuWKiIqKEkII8eWXX4oDBw6IW7duif3794tq1aqJUaNGSeXWrl0rFAqFaNu2rQgPDxcRERHCx8dHDBw4UMqzcOFCYW9vL3777Tdx+fJlMWLECGFrayu6d+8u5Rk/frzw8PAQu3btEpcuXRJDhgwRDg4O4vHjx0IIIQ4ePCgAiCZNmoiwsDBx9uxZUblyZeHv7y/at28vzp49K44cOSKcnJzEggULCjzPnPa2a9dOnD17Vhw+fFg4OTmJ9u3bi759+4pLly6J33//XSiVShESEiKV69atm/Dx8RFHjhwRkZGRokOHDqJy5coiIyNDCCHEW2+9Jd5++22tY7311ltiwIABQgghkpOTRZUqVcTw4cPFhQsXxOXLl8XAgQNFtWrVRHp6ep52ZmVlic2bNwsA4urVqyImJkb8888/QgghfvvtN7F582Zx7do1ce7cOdG1a1fh5+cn1Gq1EEKIp0+fikqVKomWLVuKo0ePiuvXr4tNmzaJ48ePi5SUFDFlyhRRs2ZNERMTI2JiYkTKvz9UAMTWrVufW8fLUtDvkhBF//5mQFREugZEFT/Y8UIXIjJ+Bf0Rz34PfP5Lp07adVhZFZzX3187r7Nz/vl05e/vL+rUqfPcfL/88otwcnKStteuXSsAiBs3bkhp33zzjXBzc5O2y5YtqxWkZGZmivLly0sBUVJSklAoFGLDhg1SnoyMDOHh4SEWLVokhPgvINq3b5+UZ/78+QKAuHnzppQ2cuRI0aFDhwLbn197R44cKaysrMTTp0+ltA4dOoiRI0cKIYS4du2aACCOHTsm7X/06JGwtLQUv/zyixBCiC1btggbGxuRnJwshMj+frGwsBA7d+4UQgixZs0aUa1aNaHRaKQ60tPThaWlpfjjjz/ybWvOOcfHxxd4PkIIERcXJwCIP//8UwghxPfffy9sbW2lYDK32bNni9q1a+dJfzYgel4dL4MhAiLeMiMiIp01aNAgT9rBgwfRrl07lCtXDra2thg8eDAeP36M5ORkKY+VlRVee+01abts2bLStAsJCQmIiYlB06ZNpf3m5uZax7p58yYyMzPRvHlzKU2hUKBRo0aIiorSak+tWrWkdTc3N1hZWaFSpUpaaTnHLkju9rq5ucHLyws2Oa/pz1VPVFQUzM3N0bhxY2m/k5MTqlWrJrWvc+fOMDc3lyYc37x5M2xtbdG+fXsAQEREBG7cuAFbW1vY2NjAxsYGjo6OSEtLy3O763lu3ryJgQMHolKlSrCzs5MmTb979y4AIDIyEnXr1oWjo6NO9T7LEHWUBhxUrYfkjGS4ztVv6g7LjDREfdkbAOAz6TekKi1eTCNfguRkTt1B9CI8M+wmj9zj9Qr7Ps89NVoBQ1D0Yp3rF/7OnTvo1KkT3n//fXz66adwdHREWFgYRowYofX2YIVCoVVOJpPlGSNUmJy8uV/EJ4TIk/bssWQyWb7Hft6kuvmVKayegs7l2fYplUr07t0bwcHB6N+/P4KDg9GvXz+Ym2d/JWs0GtSvXx8bNmzIU4+Li0uh7c2ta9eu8PT0xA8//AAPDw9oNBr4+voiIyMDwH/TXhSHIeooDdhDRERUylhbF7xYWBQ9b+7vqYLyGcKZM2eQlZWFJUuWoEmTJqhatSoePHigUx329vYoW7YsTp48KaVlZWUhIiJC2q5cuTKUSiXCwsKktMzMTJw5cwY+Pj7FP5FiqlGjBrKysrQGFD9+/BjXrl3Tal9AQABCQ0Nx6dIlHDx4EAEBAdK+evXq4fr163B1dUXlypW1Fnt7+3yPm/OU37ODux8/foyoqCh8/PHHaNOmDXx8fBAfH69VrlatWoiMjMSTJ08KrPfZOvPzvDqMBQMiIiIqttdeew1ZWVlYvnw5bt26hZ9//hnfffedzvVMmDABCxYswNatW3HlyhWMHj1a69061tbWGDVqFKZNm4bQ0FBcvnwZ7777LlJSUjBixAgDnpF+qlSpgu7du+Pdd99FWFgYzp8/j7fffhvlypVD9+7dpXz+/v5wc3NDQEAAvLy80KRJE2lfQEAAnJ2d0b17dxw9ehTR0dE4fPgwJkyYgHv37uV73IoVK0Imk2HHjh14+PAhkpKSpCfUVq1ahRs3buDAgQOYPHmyVrkBAwbA3d0dPXr0wLFjx3Dr1i1s3rwZJ06cAAB4eXkhOjoakZGRePToEdLT0/Mc+3l1GAsGREREVGx16tTB0qVLsXDhQvj6+mLDhg2YP3++zvVMmTIFgwcPxtChQ9G0aVPY2tqiZ8+eWnkWLFiAt956C4MGDUK9evVw48YN/PHHH3BwcDDU6RTL2rVrUb9+fXTp0gVNmzaFEAK7du3KcwtvwIABOH/+vFbvEJA9bunIkSOoUKECevXqBR8fHwwfPhypqamws7PL95jlypXD3LlzMWPGDLi5uWHs2LEwMzNDSEgIIiIi4Ovri0mTJmHx4sVa5ZRKJfbs2QNXV1d06tQJfn5+WLBggTRj/FtvvYWOHTuidevWcHFxwcaNG/Mc+3l1GAuZ0OXmrQlLTEyEvb09EhISILeQP3cMkdeMnfnWY6gxRLcXdNarnCFxDBFR8aSlpSE6Ohre3t6wyH0vjIiKrLDfpWe/vwsKKAH2EBERERExICIiIiLiY/d6kJvJ0a5KO4TXCUe9svV0uk+qMTPDgUoNpHVjJpcDnTr9t05ERGSsGBDpwcLcAtuGbgOG6l423VyJ4X3mGLhFJcPCAtiZ/1ApIiIio2LcXRREREREBsCAiIiIiEweAyI9JGckw2WOC5KVyRDWAkh+fpkclhlpuLz0LVxe+hYsM9JeXCNfguTk/950m6zDNSAiIiptOIZITymZKbDOtAYyn583N6vMvG/6NFYpKSXdAiIiouJjDxERERGZPAZERET0ygsKCkKZMmVKuhlGqVWrVpg4cWJJN+OFY0BERERkZGQyGbZt22bQOg8dOgSZTKY1mS4AbNmyBZ9++qlBj1UacQwRERG9cBkZGVAqlSXdDNKDo6NjSTfhpWAPERER6eTp06cICAiAtbU1ypYtiy+//DLPbRUvLy989tlnGDp0KOzt7fHuu+8CADZv3oyaNWtCpVLBy8sLS5Ys0ao7v56PMmXKICgoCABw+/ZtyGQybNmyBa1bt4aVlRVq166NEydOaJUJCgpChQoVYGVlhZ49e+Lx48eFnlNOvb/88gtatmwJS0tLNGzYENeuXUN4eDgaNGgAGxsbdOzYEQ8fPpTKaTQazJs3D+XLl4dKpUKdOnUQGhoq7W/atClmzJihdayHDx9CoVDg4MGDALKDxenTp6NcuXKwtrZG48aNcejQoQLb6uXlBQDo2bMnZDKZtH3z5k10794dbm5usLGxQcOGDbFv3z6tsunp6Zg+fTo8PT2hUqlQpUoVrFmzBrdv30br1q0BAA4ODpDJZBg6dCiAvLfMCqrD2DEg0oOZzAzNKzZHZLVIqF9X63QVNTIZTnr64qSnLzQy2Ytr5EtgZgb4+2cvRj4LCVGpkpyRXOCSlpVW5LypmalFyquryZMn49ixY9i+fTv27t2Lo0eP4uzZs3nyLV68GL6+voiIiMCsWbMQERGBvn37on///vjzzz8xZ84czJo1Swp2dPHRRx9h6tSpiIyMRNWqVTFgwABkZWUBAE6dOoXhw4dj9OjRiIyMROvWrfHZZ58Vqd7Zs2fj448/xtmzZ2Fubo4BAwZg+vTp+Oqrr3D06FHcvHkTn3zyiZT/q6++wpIlS/DFF1/gwoUL6NChA7p164br168DAAICArBx40YIIaQymzZtgpubG/z9/QEAw4YNw7FjxxASEoILFy6gT58+6Nixo1RHbuHh4QCAtWvXIiYmRtpOSkpCp06dsG/fPpw7dw4dOnRA165dcffuXans4MGDERISgq+//hpRUVH47rvvYGNjA09PT2zevBkAcPXqVcTExOCrr77K9/gF1WHsZOLZT4kKlJiYCHt7eyQkJMDOzu65+b1mvNg5LW4v6PxC6yeiFy8tLQ3R0dHw9vaGhYWFlC6bW/A/S52qdMLOgf/9fbEOtEZKZv7vv/Cv6I9DQw9J2y6LXfAo5VGefGJ20b8Gnj59CicnJwQHB6N3794AgISEBHh4eODdd9/FsmXLAGT3YtStWxdbt26VygYEBODhw4fYs2ePlDZ9+nTs3LkTly5dApDdQ7R161b06NFDylOmTBksW7YMQ4cOxe3bt+Ht7Y3Vq1djxIgRAIDLly+jZs2aiIqKQvXq1TFw4EDEx8dj9+7dUh39+/dHaGhonvExOfKrNyQkBAMGDMD+/fvxxhtvAAAWLFiAoKAgXLlyBQBQrlw5jBkzBh9++KFUV6NGjdCwYUN88803ePjwITw8PHDgwAG0bNkSANCsWTO0aNECixYtws2bN1GlShXcu3cPHh4eUh1t27ZFo0aNEBgYmG9787tO+alZsyZGjRqFsWPH4tq1a6hWrRr27t2Ltm3b5sl76NAhtG7dGvHx8VoD0Fu1aoU6depg2bJlz62jpBT0uwQU/fub/9cTEVGR3bp1C5mZmWjUqJGUZm9vj2rVquXJ26BBA63tqKgoNG/eXCutefPmuH79OtRqtU7tqFWrlrRetmxZAEBcXJx0nKZNm2rlz71dlHrd3NwAAH5+flppOcdJTEzEgwcP8j2nqKgoAICLiwvatWuHDRs2AACio6Nx4sQJBAQEAADOnj0LIQSqVq0KGxsbaTl8+DBu3rxZpDbnSE5OxvTp01GjRg2UKVMGNjY2uHLlitRDFBkZCblcLvVM6cMQdZRWHFRNRFTKJM1MKnCf3EyutR03Na7AvGYy7f95b0+4Xax2AZBu/chy3fLP72aDtbV1njzPKyeTyfKkZWbmfQOuQqHQKgNkj+cpqC1FlV+9udNyjpM7X47c5xkQEIAJEyZg+fLlCA4ORs2aNVG7dm2pzXK5HBEREZDLtT9bXW9DTZs2DX/88Qe++OILVK5cGZaWlujduzcyMjIAAJaWljrVlx9D1FFasYdID8kZyaj4eUU8snkE4aL71B0RXw9ExNcDX4mpO1xcshdO3UFkONZK6wIXC3OLIue1VFgWKa8uXnvtNSgUCpw+fVpKS0xMLHC8y7Nq1KiBsLAwrbTjx4+jatWqUjDg4uKCmJgYaf/169eRouMr8WvUqIGTJ09qpeXeNgQ7Ozt4eHjke04+Pj7Sdo8ePZCWlobQ0FAEBwfj7bfflvbVrVsXarUacXFxqFy5stbi7u5e4LEVCkWeXrWjR49i6NCh6NmzJ/z8/ODu7o7bt29L+/38/KDRaHD48OF868x5CrCw3rrn1WHM2EOkp0cpj+Cc7KxTMJTDKTXR8A0qIY/yDkcgoleYra0thgwZgmnTpsHR0RGurq6YPXs2zMzM8vSU5DZlyhQ0bNgQn376Kfr164cTJ05gxYoV+Pbbb6U8b7zxBlasWIEmTZpAo9Hggw8+0OqhKYrx48ejWbNmWLRoEXr06IE9e/ZoPfllSNOmTcPs2bPx2muvoU6dOli7di0iIyOlW2RAdk9Z9+7dMWvWLERFRWHgwIHSvqpVqyIgIACDBw/GkiVLULduXTx69AgHDhyAn58fOnXqlO9xvby8sH//fjRv3hwqlQoODg6oXLkytmzZgq5du0Imk2HWrFlavVleXl4YMmQIhg8fjq+//hq1a9fGnTt3EBcXh759+6JixYqQyWTYsWMHOnXqBEtLyzy9VM+rw5ixh4iIiHSydOlSNG3aFF26dEHbtm3RvHlz+Pj45BnMmlu9evXwyy+/ICQkBL6+vvjkk08wb9486fFuAFiyZAk8PT3x+uuvY+DAgZg6dSqsrKx0al+TJk2wevVqLF++HHXq1MGePXvw8ccf63OqzzV+/HhMmTIFU6ZMgZ+fH0JDQ7F9+3ZUqVJFK19AQADOnz+Pli1bokKFClr71q5di8GDB2PKlCmoVq0aunXrhlOnTsHT07PA4y5ZsgR79+6Fp6cn6tatCwD48ssv4eDggGbNmqFr167o0KED6tWrp1Vu5cqV6N27N0aPHo3q1avj3XffRfK/XfzlypXD3LlzMWPGDLi5uWHs2LH5HruwOowZnzIromdHqcst5HCd64rkwH9/AJIA5Op1LugpM8uMNER9mf1khs+k35CqLPwPSEFKw1NmyclAzj8PSUnZs94TUdEV9mSMMUlOTka5cuWwZMkS6QktopfJEE+Z8ZYZERHp5Ny5c7hy5QoaNWqEhIQEzJs3DwDQvXv3Em4Zkf4YEBERkc6++OILXL16FUqlEvXr18fRo0fh7Oxc0s0i0hsDIiIi0kndunURERFR0s0gMigGRHowk5mhrkddXK54GdWdq8NMh3krNDIZzrtXkdaNmZkZkPPeNU7dQURExowBkR4sFZYIGxMGjNG9bLpChe5DvjR8o0qApSXw7xQ6RERERo3/1xMREZHJY0BEREREJo8BkR5SMlNQfVF13HO8B42XBtDhrfIWmWkIWzkcYSuHwyLTuKfuSEkBvLyyFx3frE9ERFSqcAyRHoQQ+CvhL5SPLw/EA9Dh1ZYyAZRPjJPWjZkQwJ07/60TEREZK/YQERGRQc2ZMwdubm6QyWTYtm1bSTfH5Ny+fRsymQyRkZEl3RSjwoCIiIgMJioqCnPnzsX333+PmJgYvPnmm7h06RLeeusteHl5QSaTYdmyZYXWMX/+fMhkMkycOPGltPllCAoKQpkyZQxe79ChQ9GjRw+tNE9PT8TExMDX19fgx3uVMSAiIiKDuXnzJoDsaTzc3d2hUqmQkpKCSpUqYcGCBXB3dy+0fHh4OFatWoVatWq9jOa+kuRyOdzd3WFuzlExuijRgOjIkSPo2rUrPDw88u1aFUJgzpw58PDwgKWlJVq1aoVLly5p5UlPT8e4cePg7OwMa2trdOvWDffu3dPKEx8fj0GDBsHe3h729vYYNGgQ/vnnnxd8dkREr6bffvsNfn5+sLS0hJOTE9q2bYvk5GTMmTMHXbt2BQCYmZlB9u/LZxs2bIjFixejf//+UKlUBdablJSEgIAA/PDDD3BwcHhuO+bMmYM6dergxx9/RIUKFWBjY4NRo0ZBrVZj0aJFcHd3h6urKz7//HOtcnfv3kX37t1hY2MDOzs79O3bF3///TcA4OrVq5DJZLhy5YpWmaVLl8LLyws586FfvnwZnTp1go2NDdzc3DBo0CA8evQo33YeOnQIw4YNQ0JCAmQyGWQyGebMmQMAWL9+PRo0aABbW1u4u7tj4MCBiIuL0yp/6dIldO7cGXZ2drC1tUXLli1x8+ZNzJkzB+vWrcP//vc/qd5Dhw7le8usoDroPyUaECUnJ6N27dpYsWJFvvsXLVqEpUuXYsWKFQgPD4e7uzvatWuHp0+fSnkmTpyIrVu3IiQkBGFhYUhKSkKXLl2gVqulPAMHDkRkZCRCQ0MRGhqKyMhIDBo06IWfHxGRXpKTC17S0oqeNzW1aHl1EBMTgwEDBmD48OGIiorCoUOH0KtXLwghMHXqVKxdu1bKFxMTo1PdY8aMQefOndG2bdsil7l58yZ2796N0NBQbNy4ET/++CM6d+6Me/fu4fDhw1i4cCE+/vhjnDx5EkD2P9o9evTAkydPcPjwYezduxc3b95Ev379AADVqlVD/fr1sWHDBq3jBAcHY+DAgZDJZIiJiYG/vz/q1KmDM2fOIDQ0FH///Tf69u2bbxubNWuGZcuWwc7OTrouU6dOBQBkZGTg008/xfnz57Ft2zZER0dj6NChUtn79+/j9ddfh4WFBQ4cOICIiAgMHz4cWVlZmDp1Kvr27YuOHTtK9TZr1izP8Qurg/5Tov1pb775Jt5888189wkhsGzZMnz00Ufo1asXAGDdunVwc3NDcHAwRo4ciYSEBKxZswY///yz9Au0fv16eHp6Yt++fejQoQOioqIQGhqKkydPonHjxgCAH374AU2bNsXVq1dRrVo1ndstk8lQ3aU6brjfQCWHSjCTFT2uFDLgmlMFad2YyWRAjRr/rRORgdjYFLyvUydg587/tl1dC37vhb8/cOjQf9teXkB+vRg6PCYaExODrKws9OrVCxUrVgQA+Pn5Sftzxsk879ZYbiEhITh79izCdXz9vUajwY8//ghbW1vUqFEDrVu3xtWrV7Fr1y6YmZmhWrVqWLhwIQ4dOoQmTZpg3759uHDhAqKjo+Hp6QkA+Pnnn1GzZk2Eh4ejYcOGCAgIwIoVK/Dpp58CAK5du4aIiAj89NNPAICVK1eiXr16CAwMlNrx448/wtPTE9euXUPVqlW12qhUKmFvbw+ZTJbnugwfPlxar1SpEr7++ms0atQISUlJsLGxwTfffAN7e3uEhIRAoVAAgFb9lpaWSE9PL/R6P68OylZqxxBFR0cjNjYW7du3l9JUKhX8/f1x/PhxAEBERAQyMzO18nh4eMDX11fKc+LECdjb20vBEAA0adIE9vb2Up78pKenIzExUWvJYaWwQsTECFSOqQyzy2aAVdHPK01hgfbvfIv273yLNIVF0QuWQlZWwKVL2YuVDteAiIxX7dq10aZNG/j5+aFPnz744YcfEB8fX6w6//rrL0yYMAHr16+HhYVufxe9vLxga2srbbu5uaFGjRpac0y6ublJt6GioqLg6ekpBUMAUKNGDZQpUwZRUVEAgP79++POnTtSr9KGDRtQp04d1Pj3P8CIiAgcPHgQNjY20lK9enUA0Pk21Llz59C9e3dUrFgRtra2aNWqFYDs23oAEBkZiZYtW0qBjD4MUYcpKLUjrmJjYwFk/yA/y83NDXf+fflNbGwslEplnnvNbm5uUvnY2Fi4urrmqd/V1VXKk5/58+dj7ty5xToHIiK9JCUVvE8u197ONd5ES+5Zl2/f1rtJ/x1ejr179+L48ePYs2cPli9fjo8++ginTp2Ct7e3XnVGREQgLi4O9evXl9LUajWOHDmCFStWID09HfLc5/2v3F/yMpks3zSNRgMg++6DLJ8u7WfTy5Yti9atWyM4OBhNmjTBxo0bMXLkSCmvRqNB165dsXDhwjz1lC1btohnnT1spH379mjfvj3Wr18PFxcX3L17Fx06dEBGRgaA7B6g4jJEHaag1PYQ5cj9g1vQD3NheZ73w5+fmTNnIiEhQVr++usvHVtORKQna+uCl9w9KIXlzf1FWFA+HclkMjRv3hxz587FuXPnoFQqsXXrVr1Pt02bNvjzzz8RGRkpLQ0aNEBAQAAiIyMLDIb0UaNGDdy9e1frb/rly5eRkJAAHx8fKS0gIACbNm3CiRMncPPmTfTv31/aV69ePVy6dAleXl6oXLmy1mJdwPVUKpVaY1sB4MqVK3j06BEWLFiAli1bonr16nkGVNeqVQtHjx5FZmZmkevN7Xl1ULZSGxDl3A/N3YsTFxcn9Rq5u7sjIyMjT3dt7jw5Tw886+HDh3l6n56lUqlgZ2enteRIyUxB/WX1caPsDWhq6D51x57Vo7Fn9ehXYuqOmjWzF07dQWQaTp06hcDAQJw5cwZ3797Fli1b8PDhQ61gIreMjAwp0MnIyMD9+/cRGRmJGzduAABsbW3h6+urtVhbW8PJycng79Jp27YtatWqhYCAAJw9exanT5/G4MGD4e/vjwYNGkj5evXqhcTERIwaNQqtW7dGuXLlpH1jxozBkydPMGDAAJw+fRq3bt3Cnj17MHz48AKDEy8vLyQlJWH//v149OgRUlJSUKFCBSiVSixfvhy3bt3C9u3bpXFLOcaOHYvExET0798fZ86cwfXr1/Hzzz/j6tWrUr0XLlzA1atX8ejRo3yDnufVQdlKbUDk7e0Nd3d37N27V0rLyMjA4cOHpVH09evXh0Kh0MoTExODixcvSnmaNm2KhIQEnD59Wspz6tQpJCQk5DsavyiEELjy8Aoqx1aGWZSZzlN3VH18F1Uf330lpu64fDl74dQdRKbBzs4OR44cQadOnVC1alV8/PHHWLJkSYEPyADAgwcPULduXdStWxcxMTH44osvULduXbzzzjsvseXZcl7x4uDggNdffx1t27ZFpUqVsGnTJq18dnZ26Nq1K86fP4+AgACtfR4eHjh27BjUajU6dOgAX19fTJgwAfb29lpjl57VrFkzvP/+++jXrx9cXFywaNEiuLi4ICgoCL/++itq1KiBBQsW4IsvvtAq5+TkhAMHDiApKQn+/v6oX78+fvjhB+m24Lvvvotq1aqhQYMGcHFxwbFjx/Ic+3l1UDaZECX3VZaUlCT9h1C3bl0sXboUrVu3hqOjIypUqICFCxdi/vz5WLt2LapUqYLAwEAcOnQIV69elQbRjRo1Cjt27EBQUBAcHR0xdepUPH78GBEREVI365tvvokHDx7g+++/BwC89957qFixIn7//fcitzUxMRH29vZISEiA3EIO17muSA7893HVJAC5ekm9ZuzMUwcAWGakIerL3gAAn0m/IVWp38Dq2ws661XOkJKT/3sYJilJr553IpOWlpaG6OhoeHt76zyYmIj+U9jv0rPf38/e7cmtRAdVnzlzBq1bt5a2J0+eDAAYMmQIgoKCMH36dKSmpmL06NGIj49H48aNsWfPHq0nCr788kuYm5ujb9++SE1NRZs2bRAUFKR1z3nDhg0YP3689DRat27dCnz3EREREZmeEg2IWrVqhcI6qHLe5pnzRs/8WFhYYPny5Vi+fHmBeRwdHbF+/friNJWIiIheYaV2DBERERHRy8KAiIiIiExeqX0xY2kmk8ngae+Jew734GHnofPUHffsXKV1YyaTAf++uZ9TdxARkVFjQKQHK4UVrky/AkzXvWyawgItRv1o+EaVACsrg7z4loiIqMTxlhkRERGZPAZEREREZPIYEOkhNTMVLb5pgctel6FpoAFSi15WlZmO/62bhP+tmwRVZvqLa+RLkJoKNGyYvaTqcA2IiIhKGwZEetAIDc49OIcad2rALMIM0BS9rJkQqB17HbVjr8PMyOe70GiAM2eyF40O14CIKD9Dhw5Fjx49SroZVEJyplUpKQyIiIioVPjqq68QFBRU7HomTJiA+vXrQ6VSoU6dOvnmEULgiy++QNWqVaFSqeDp6YnAwECtPIcPH0b9+vVhYWGBSpUq4bvvvstTz+bNm1GjRg2oVCrUqFEDW7duLXb7S4tDhw5BJpPhn3/+MWi9c+bMyfdziYmJKXROvBeNAREREZUK9vb2KFOmTLHrEUJg+PDh6NevX4F5JkyYgNWrV+OLL77AlStX8Pvvv6NRo0bS/ujoaHTq1AktW7bEuXPn8OGHH2L8+PHYvHmzlOfEiRPo168fBg0ahPPnz2PQoEHo27cvTp06VexzMEXu7u5QqVQldnwGREREpJNWrVph3LhxmDhxIhwcHODm5oZVq1YhOTkZw4YNg62tLV577TXs3r1bKqNWqzFixAh4e3vD0tIS1apVw1dffaVVb+5bZqGhoWjRogXKlCkDJycndOnSBTdv3nxu+77++muMGTMGlSpVynd/VFQUVq5cif/973/o1q0bvL29UadOHbRt21bK891336FChQpYtmwZfHx88M4772D48OFas9EvW7YM7dq1w8yZM1G9enXMnDkTbdq0wbJlywpsW1BQEMqUKYMdO3agWrVqsLKyQu/evZGcnIx169bBy8sLDg4OGDduHNRqtVQuPj4egwcPhoODA6ysrPDmm2/i+vXrAICEhARYWloiNDRU61hbtmyBtbU1kpKSAAD3799Hv3794ODgACcnJ3Tv3h23C3h3yu3bt6W5Rh0cHCCTyTB06FAARftc7t27h/79+8PR0RHW1tZo0KABTp06haCgIMydOxfnz5+HTCaDTCaTegVz3zIrqI4XhQEREVFpk1zIkqZD3twPOxSUTw/r1q2Ds7MzTp8+jXHjxmHUqFHo06cPmjVrhrNnz6JDhw4YNGgQUlJSAAAajQbly5fHL7/8gsuXL+OTTz7Bhx9+iF9++aXgy5CcjMmTJyM8PBz79++HmZkZevbsCU0xBy3+/vvvqFSpEnbs2AFvb294eXnhnXfewZMnT6Q8J06ckCYEz9GhQwecOXMGmZmZheY5fvx4ocdPSUnB119/jZCQEISGhuLQoUPo1asXdu3ahV27duHnn3/GqlWr8Ntvv0llhg4dijNnzmD79u04ceIEhBDo1KkTMjMzYW9vj86dO2PDhg1axwkODkb37t1hY2ODlJQUtG7dGjY2Njhy5AjCwsJgY2ODjh07IiMjI08bPT09pd6wq1evIiYmRgpgn/e5JCUlwd/fHw8ePMD27dtx/vx5TJ8+HRqNBv369cOUKVNQs2ZNxMTEICYmJt+evMLqeFH4YkYiotLGppB9nQDsfGbbFUBKAXn9ARx6ZtsLwKN88unxfEft2rXx8ccfAwBmzpyJBQsWwNnZGe+++y4A4JNPPsHKlStx4cIFNGnSBAqFAnPnzpXKe3t74/jx4/jll1/Qt2/ffI/x1ltvaW2vWbMGrq6uuHz5Mnx9fXVv9L9u3bqFO3fu4Ndff8VPP/0EtVqNSZMmoXfv3jhw4AAAIDY2Fm5ublrl3NzckJWVhUePHqFs2bIF5omNjS30+JmZmVi5ciVee+01AEDv3r3x888/4++//4aNjQ1q1KiB1q1b4+DBg+jXrx+uX7+O7du349ixY2jWrBkAYMOGDfD09MS2bdvQp08fBAQEYPDgwUhJSYGVlRUSExOxc+dOKagJCQmBmZkZVq9eDdm/UwusXbsWZcqUwaFDh/IEdnK5HI6OjgAAV1dXrVuZz/tcgoOD8fDhQ4SHh0t1VK5cWcpvY2MDc3NzuLu7F3iNnlfHi8AeIj05WznjkfUjCGfd/5I8trTDY0u7F9Cql8/ZOXshItNSq1YtaV0ul8PJyQl+fn5SWk6gEBcXJ6V99913aNCgAVxcXGBjY4MffvgBd+/eLfAYN2/exMCBA1GpUiXY2dnB29sbAKQyb775JmxsbGBjY4OaNWsWue0ajQbp6en46aef0LJlS7Rq1Qpr1qzBwYMHcfXqVSmfLNecROLfJ4OfTc8vT+603KysrKRgCMi+Vl5eXrCxsdFKy7l2UVFRMDc3R+PGjaX9Tk5OqFatGqKiogAAnTt3hrm5ObZv3w4ge7C3ra2tFOhERETgxo0bsLW1la6Zo6Mj0tLSinQb8lnP+1wiIyNRt25dKZDRhyHq0BV7iPRgrbTGnY/uAB/pXjZVaYH644MN36gSYG0NPHxY0q0gegUlFbJPnms7Lt9c2XL/y3tbr9bkS6FQaG3LZDKttJygIOcWxy+//IJJkyZhyZIlaNq0KWxtbbF48eJCx4R07doVnp6e+OGHH+Dh4QGNRgNfX1/pFs/q1auR+u9L0HK3pzBly5aFubk5qlatKqX5+PgAyP5Sr1atGtzd3fP09MTFxcHc3BxOTk4AUGCe3L1GuT3v2uWk5Vw7UcArWp4NvpRKJXr37o3g4GD0798fwcHB6NevH8zNs7/mNRoN6tevn+e2GgC4uLgU2t7cnve5WFpa6lRffgxRh64YEBERlTbWpSCvgR09ehTNmjXD6NGjpbTCeiYeP36MqKgofP/992jZsiUAICwsTCtPuXLl9GpL8+bNkZWVhZs3b0o9NdeuXQMAVPx3xuqmTZvi999/1yq3Z88eNGjQQApemjZtir1792LSpElaeXJuaxlKjRo1kJWVhVOnTkl1P378GNeuXZMCOQAICAhA+/btcenSJRw8eBCffvqptK9evXrYtGkTXF1dYWdXtDsUSqUSALQGdxflc6lVqxZWr16NJ0+e5NvDo1QqterMz/PqeBF4y4yIiF64ypUr48yZM/jjjz9w7do1zJo1C+Hh4QXmz3kSatWqVbhx4wYOHDiAyZMnF+lYN27cQGRkJGJjY5GamorIyEhERkZKPRht27ZFvXr1MHz4cJw7dw4REREYOXIk2rVrJ/Uavf/++7hz5w4mT56MqKgo/Pjjj1izZg2mTp0qHWfChAnYs2cPFi5ciCtXrmDhwoXYt28fJk6cqP+FykeVKlXQvXt3vPvuuwgLC8P58+fx9ttvo1y5cujevbuUz9/fH25ubggICICXlxeaNGki7QsICICzszO6d++Oo0ePIjo6GocPH8aECRNw7969fI9bsWJFyGQy7NixAw8fPkRSUlKRPpcBAwbA3d0dPXr0wLFjx3Dr1i1s3rwZJ06cAAB4eXkhOjoakZGRePToEdLT887a8Lw6XgQGRHpIzUxF+1XtEVk9Emp/tc5Td4QEz0BI8IxXYuqOVq2yF07dQUSFef/999GrVy/069cPjRs3xuPHj7V6i3IzMzNDSEgIIiIi4Ovri0mTJmHx4sVFOtY777yDunXr4vvvv8e1a9dQt25d1K1bFw8ePJDq/v333+Hs7IzXX38dnTt3ho+PD0JCQqQ6vL29sWvXLhw6dAh16tTBp59+iq+//lprQHGzZs0QEhKCtWvXolatWggKCsKmTZu0xvoYytq1a1G/fn106dIFTZs2hRACu3btynObcsCAATh//jwCAgK0yltZWeHIkSOoUKECevXqBR8fHwwfPhypqakF9hiVK1cOc+fOxYwZM+Dm5oaxY8cW6XNRKpXYs2cPXF1d0alTJ/j5+WHBggWQy7Pv97711lvo2LEjWrduDRcXF2zcuDHPsZ9Xx4sgEwXdnCQtiYmJsLe3R0JCAuQWcrjOdUVy4L/PqyYhT1e014ydeeoAAMuMNER92RsA4DPpN6QqLfRqz+0FnfUqZ0jJyUDOGMCkpOwxRURUdGlpaYiOjoa3tzcsLPT7W0BEhf8uPfv9XdjtQvYQERERkcljQEREREQmjwERERERmTwGRERERGTyGBAREZUwPttCVDyG+B1iQKQnK4UVkhXJEFa6fwgpChVSFKoX0KqXz8oqeyEi3eU8Qpzf5JpEVHQ5kwjr8sby3Pimaj1YK63xcM5DYI7uZVOVFqgxebPB21QSrK2zH70nIv2Ym5vDysoKDx8+hEKhgJkZ/0cl0oUQAikpKYiLi0OZMmWK9Z4iBkRERCVEJpOhbNmyiI6Oxp07d0q6OURGq0yZMnB3dy9WHQyIiIhKkFKpRJUqVXjbjEhPCoXCIG+wZkCkh7SsNPRf3x8fffUR6pWtB/kWOVDEl8yqsjKwcmsgAGBUzw+Rbq58gS19sdLSgJy32G/eDPBFu0T6MTMz45uqiUoYAyI9qDVq7L2+F9sitwGRAAqftFeLmUaDN26dkdaNmVoN7Nr13zoREZGx4gg+IiIiMnkMiIiIiMjkMSAiIiIik8eAiIiIiEweAyIiIiIyeQyIiIiIyOTxsXs9WCutkfx5MvC57mVTlRbw+mCH4RtVAqytAc5JSURErwL2EBEREZHJY0BEREREJo8BkR7SstIwcMNAnGh6AureaiCt6GVVWRn4Ztt8fLNtPlRZxj13UVoa0KdP9pKmwzUgIiIqbRgQ6UGtUeN/l/+HpiebQr5ZrvPUHZ2vHkPnq8deiak7fvste+HUHUREZMwYEBEREZHJY0BEREREJo8BEREREZk8BkRERERk8hgQERERkcljQEREREQmj1N36MFKYYW4T+KQPDUZVgoryKxkRS6bqlDBZ9Jv0roxs7ICkpL+WyciIjJWDIj0IJPJYK2yBvSJZ2QypCotDN6mkiCTZc9nRkREZOx4y4yIiIhMXqkOiLKysvDxxx/D29sblpaWqFSpEubNmwfNM294FkJgzpw58PDwgKWlJVq1aoVLly5p1ZOeno5x48bB2dkZ1tbW6NatG+7du6d3u9Kz0vHOr+8g7I0wqAergfSil1VmZeKLnV/ii51fQpmVqXcbSoP0dGDo0OwlXYdrQEREVNqU6oBo4cKF+O6777BixQpERUVh0aJFWLx4MZYvXy7lWbRoEZYuXYoVK1YgPDwc7u7uaNeuHZ4+fSrlmThxIrZu3YqQkBCEhYUhKSkJXbp0gVrP+SayNFnYGLkRLQ62gPxnOZBV9LJyjRq9L+5H74v7IdcY93wXWVnAunXZS5YO14CIiKi0KdVjiE6cOIHu3bujc+fOAAAvLy9s3LgRZ86cAZDdO7Rs2TJ89NFH6NWrFwBg3bp1cHNzQ3BwMEaOHImEhASsWbMGP//8M9q2bQsAWL9+PTw9PbFv3z506NChZE6OiIiISo1S3UPUokUL7N+/H9euXQMAnD9/HmFhYejUqRMAIDo6GrGxsWjfvr1URqVSwd/fH8ePHwcAREREIDMzUyuPh4cHfH19pTxERERk2kp1D9EHH3yAhIQEVK9eHXK5HGq1Gp9//jkGDBgAAIiNjQUAuLm5aZVzc3PDnTt3pDxKpRIODg558uSUz096ejrSnxkYk5iYaJBzIiIiotKnVPcQbdq0CevXr0dwcDDOnj2LdevW4YsvvsC6deu08slk2u8BEkLkScvteXnmz58Pe3t7afH09NT/RIiIiKhUK9UB0bRp0zBjxgz0798ffn5+GDRoECZNmoT58+cDANzd3QEgT09PXFyc1Gvk7u6OjIwMxMfHF5gnPzNnzkRCQoK0/PXXX4Y8NSIiIipFSnVAlJKSAjMz7SbK5XLpsXtvb2+4u7tj79690v6MjAwcPnwYzZo1AwDUr18fCoVCK09MTAwuXrwo5cmPSqWCnZ2d1kJERESvplI9hqhr1674/PPPUaFCBdSsWRPnzp3D0qVLMXz4cADZt8omTpyIwMBAVKlSBVWqVEFgYCCsrKwwcOBAAIC9vT1GjBiBKVOmwMnJCY6Ojpg6dSr8/Pykp850ZaWwwu2Zt/Ho3UdwsnLSeeqOeuM2SOvGzMoKiIv7b52IiMhYleqAaPny5Zg1axZGjx6NuLg4eHh4YOTIkfjkk0+kPNOnT0dqaipGjx6N+Ph4NG7cGHv27IGtra2U58svv4S5uTn69u2L1NRUtGnTBkFBQZDL5Xq1SyaTwcXGBbDRqzCeWNnrddzSRiYDXFxKuhVERETFJxNCiJJuhDFITEyEvb09EhISinT7zGvGzhfantsLOr/Q+omIiF4FRf3+LtVjiEqr9Kx0TNg2AYe7HIZ6lO5Td8zbsxLz9qx8JabuGDMme+HUHUREZMwYEOkhS5OF1eGr4b/TH/LvdJ+6Y/C5nRh8bucrMXXHt99mL5y6g4iIjBkDIiIiIjJ5DIiIiIjI5DEgIiIiIpPHgIiIiIhMHgMiIiIiMnkMiIiIiMjkleo3VZdWlgpLXJpyCff63YOHrQfMLIseV6YplGjx/hpp3ZhZWgLR0f+tExERGSsGRHowk5nBy9ELcNS9rJCZ4Z69m8HbVBLMzAAvr5JuBRERUfExIKKX6kVOacLpTIiISF8cQ6SHDHUGZuycgUN9DkE9RQ1kFL2sQp2JmQd/xMyDP0KhNu6pOzIygGnTspcMHa4BERFRacOASA+Z6kwsP74crX5rBflSOaBDXGOuVmPk6S0YeXoLzNXGPXVHZibwxRfZS6Zxx3ZERGTiGBARERGRyWNARERERCaPARERERGZPAZEREREZPIYEBEREZHJY0BEREREJo8vZtSDpcISp8edxo0ON1DJsZLOU3e0G/6NtG7MLC2Bixf/WyciIjJWDIj0YCYzQ033moC77mWFzAzXXSoavlElwMwMqFmzpFtBRERUfLxlRkRERCZPr4AoOmeKcxOVoc7Ap3s/xaGhh6D+RPepOyaGbcDEsA2vxNQdc+ZkL5y6g4iIjJleAVHlypXRunVrrF+/HmlpaYZuU6mXqc7EgkML0GpdK8g/1X3qjonHNmLisY2vxNQdc+dmL5y6g4iIjJleAdH58+dRt25dTJkyBe7u7hg5ciROnz5t6LYRERERvRR6BUS+vr5YunQp7t+/j7Vr1yI2NhYtWrRAzZo1sXTpUjx8+NDQ7SQiIiJ6YYo1qNrc3Bw9e/bEL7/8goULF+LmzZuYOnUqypcvj8GDByMmJsZQ7SQiIiJ6YYoVEJ05cwajR49G2bJlsXTpUkydOhU3b97EgQMHcP/+fXTv3t1Q7SQiIiJ6YfR6D9HSpUuxdu1aXL16FZ06dcJPP/2ETp06wcwsO77y9vbG999/j+rVqxu0sUREREQvgl4B0cqVKzF8+HAMGzYM7u75v52wQoUKWLNmTbEaR0RERPQy6BUQXb9+/bl5lEolhgwZok/1pZ6FuQUOjTyEy80uo5pzNcgt5EUum26uQLfBS6V1Y2ZhAeQ8XGhhUbJtISIiKg69AqK1a9fCxsYGffr00Ur/9ddfkZKS8soGQjnkZnI0rNAQqKB7WY2ZHBfKVjV8o0qAXA40bFjSrSAiIio+vQZVL1iwAM7OznnSXV1dERgYWOxGEREREb1MegVEd+7cgbe3d570ihUr4u7du8VuVGmXoc7AkkNLcGjMIWQtzNJ56o73Tm3Ge6c2vxJTdyxenL1w6g4iIjJmet0yc3V1xYULF+Dl5aWVfv78eTg5ORmiXaVapjoTn+z9BMnfJmcnjAWgLFpZc7UaHx5aCwD4uW5nZMqNdxxRZiYwfXr2+ujRgLKI14CIiKi00auHqH///hg/fjwOHjwItVoNtVqNAwcOYMKECejfv7+h20hERET0QunVQ/TZZ5/hzp07aNOmDczNs6vQaDQYPHgwxxARERGR0dErIFIqldi0aRM+/fRTnD9/HpaWlvDz80PFihUN3T4iIiKiF06vgChH1apVUbXqq/EIOREREZkuvQIitVqNoKAg7N+/H3FxcdBoNFr7Dxw4YJDGEREREb0MegVEEyZMQFBQEDp37gxfX1/IZDJDt4uIiIjopdErIAoJCcEvv/yCTp06Gbo9RsHC3AI7h+9EpF8k/Fz9dJ66o/+AQGndmFlYAAcP/rdORERkrPQeVF25cmVDt8VoyM3kaPVaK+A13ctqzOQ4WaGWwdtUEuRyoFWrkm4FERFR8en1HqIpU6bgq6++ghDC0O0hIiIieun06iEKCwvDwYMHsXv3btSsWRMKhfatny1bthikcaVVpjoTq0+tRo2tNdCiQgvI35cDRbz7Za7OwoDzoQCAjbU7IkterAf9SlRmJrBqVfb6e+8BCuO+A0hERCZMr2/jMmXKoGfPnoZui9HIUGdg6s6pSP7i36k7hqPIAZFCnYVP934HAPjNt61RB0QZGcDYsdnrQ4cyICIiIuOl17fx2rVrDd0OIiIiohKj1xgiAMjKysK+ffvw/fff4+nTpwCABw8eICkpyWCNIyIiInoZ9OohunPnDjp27Ii7d+8iPT0d7dq1g62tLRYtWoS0tDR89913hm4nERER0QujVw/RhAkT0KBBA8THx8PS0lJK79mzJ/bv32+wxhERERG9DHo/ZXbs2DEolUqt9IoVK+L+/fsGaRgRERHRy6JXD5FGo4Farc6Tfu/ePdja2ha7Uc+6f/8+3n77bTg5OcHKygp16tRBRESEtF8IgTlz5sDDwwOWlpZo1aoVLl26pFVHeno6xo0bB2dnZ1hbW6Nbt264d++eQdtJRERExkuvgKhdu3ZYtmyZtC2TyZCUlITZs2cbdDqP+Ph4NG/eHAqFArt378bly5exZMkSlClTRsqzaNEiLF26FCtWrEB4eDjc3d3Rrl07aaA3AEycOBFbt25FSEgIwsLCkJSUhC5duuQb1BWFylyFXwN+Rfj34VBvVwOqopfNMFdgWO/ZGNZ7NjKMfOoOlQrYsSN7UelwDYiIiEobmdDjddMPHjxA69atIZfLcf36dTRo0ADXr1+Hs7Mzjhw5AldXV4M0bsaMGTh27BiOHj2a734hBDw8PDBx4kR88MEHALJ7g9zc3LBw4UKMHDkSCQkJcHFxwc8//4x+/fpJ7ff09MSuXbvQoUOHIrUlMTER9vb2SEhIgJ2d3XPze83YWcSz1M/tBZ1faP0vyou8LsZ6TYiI6MUp6ve3Xj1EHh4eiIyMxNSpUzFy5EjUrVsXCxYswLlz5wwWDAHA9u3b0aBBA/Tp0weurq6oW7cufvjhB2l/dHQ0YmNj0b59eylNpVLB398fx48fBwBEREQgMzNTK4+Hhwd8fX2lPPlJT09HYmKi1kJERESvJr1fk2xpaYnhw4dj+PDhhmyPllu3bmHlypWYPHkyPvzwQ5w+fRrjx4+HSqXC4MGDERsbCwBwc3PTKufm5oY7d+4AAGJjY6FUKuHg4JAnT075/MyfPx9z587Nd1+mOhPBZ4Px2u7X0KR8E5gPMtdp6o4elw8BALbVaGXUb6rOzAQ2bMheDwjgm6qJiMh46fVt/NNPPxW6f/DgwXo1JjeNRoMGDRogMDAQAFC3bl1cunQJK1eu1DqGTCbTKieEyJOW2/PyzJw5E5MnT5a2ExMT4enpCSB76o7R20YjOfDfqTv6QaepO77YtQwAsLNaC6MOiDIygGHDstf79GFARERExkuvb+MJEyZobWdmZiIlJQVKpRJWVlYGC4jKli2LGjVqaKX5+Phg8+bNAAB3d3cA2b1AZcuWlfLExcVJvUbu7u7IyMhAfHy8Vi9RXFwcmjVrVuCxVSoVVBwpTEREZBL0GkMUHx+vtSQlJeHq1ato0aIFNm7caLDGNW/eHFevXtVKu3btGipWrAgA8Pb2hru7O/bu3Svtz8jIwOHDh6Vgp379+lAoFFp5YmJicPHixUIDIiIiIjIdBrtfU6VKFSxYsABvv/02rly5YpA6J02ahGbNmiEwMBB9+/bF6dOnsWrVKqxatQpA9q2yiRMnIjAwEFWqVEGVKlUQGBgIKysrDBw4EABgb2+PESNGYMqUKXBycoKjoyOmTp0KPz8/tG3b1iDtJCIiIuNm0AEscrkcDx48MFh9DRs2xNatWzFz5kzMmzcP3t7eWLZsGQICAqQ806dPR2pqKkaPHo34+Hg0btwYe/bs0XpB5Jdffglzc3P07dsXqampaNOmDYKCgiCXyw3WViIiIjJeegVE27dv19oWQiAmJgYrVqxA8+bNDdKwHF26dEGXLl0K3C+TyTBnzhzMmTOnwDwWFhZYvnw5li9fbtC2ERER0atBr4CoR48eWtsymQwuLi544403sGTJEkO0i4iIiOil0Ssg0mg0hm6HUVGZq7Cu7zqccDqBRuUaQa4q+q23DHMFRnefIa0bM5UK+OWX/9aJiIiMlfG+BKcEmZuZo3ft3kBt3cuqzeTYVb2F4RtVAszNs98/REREZOz0CoiefWHh8yxdulSfQxARERG9NHoFROfOncPZs2eRlZWFatWqAch+P5BcLke9evWkfM97W7SxytJkYduf21Buf7nsW2ZvyYt8JeUaNTpcOwEA+KNqU6jNjPdJt6wsYOvW7PWePbN7jIiIiIyRXl9hXbt2ha2tLdatWye9/Tk+Ph7Dhg1Dy5YtMWXKFIM2srRJz0rHkF+G/Dd1RxKKfCWVWZn49n8LAAA+k35DqtJ4A6L0dKBv3+z1pCQGREREZLz0elP1kiVLMH/+fK2pMBwcHPDZZ5/xKTMiIiIyOnoFRImJifj777/zpMfFxeHp06fFbhQRERHRy6RXQNSzZ08MGzYMv/32G+7du4d79+7ht99+w4gRI9CrVy9Dt5GIiIjohdJr1Md3332HqVOn4u2330ZmZmZ2RebmGDFiBBYvXmzQBhIRERG9aHoFRFZWVvj222+xePFi3Lx5E0IIVK5cGdbW1oZuHxEREdELp9ctsxwxMTGIiYlB1apVYW1tDSGEodpFRERE9NLo1UP0+PFj9O3bFwcPHoRMJsP169dRqVIlvPPOOyhTpswr/6SZUq7Etz2+RZgqDE3KN4G5suiXMVNujqmdJkrrxkypBNau/W+diIjIWOn1jTxp0iQoFArcvXsXPj4+Unq/fv0wadKkVz4gUsgVGNJwCNBQ97JZcnP85tfW8I0qAQoFMHRoSbeCiIio+PQKiPbs2YM//vgD5cuX10qvUqUK7ty5Y5CGEREREb0segVEycnJsLKyypP+6NEjqExg2vMsTRb2XNkDlzAX1CtbD/I3dZu64/XoswCAI971jH7qjj/+yF7v0IFvqiYiIuOl16Dq119/HT/99JO0LZPJoNFosHjxYrRu3dpgjSut0rPS0WdDHzQc2RDybnIgvehllVmZWPvbXKz9bS6UWZkvrpEvQXo60KVL9pKuwzUgIiIqbfT6n37x4sVo1aoVzpw5g4yMDEyfPh2XLl3CkydPcOzYMUO3kYiIiOiF0quHqEaNGrhw4QIaNWqEdu3aITk5Gb169cK5c+fw2muvGbqNRERERC+Uzj1EmZmZaN++Pb7//nvMnTv3RbSJiIiI6KXSuYdIoVDg4sWLkMlkL6I9RERERC+dXrfMBg8ejDVr1hi6LUREREQlQq9B1RkZGVi9ejX27t2LBg0a5JnDbOnSpQZpHBEREdHLoFNAdOvWLXh5eeHixYuoV68eAODatWtaeUzhVppSrsQXnb/A4YzDaFGhBeTKor9LKFNujlnt3pfWjZlSCaxY8d86ERGRsdLpG7lKlSqIiYnBwYMHAWRP1fH111/Dzc3thTSutFLIFRjVbBTQTPeyWXJz/Fyvi+EbVQIUCmDMmJJuBRERUfHpNIYo92z2u3fvRnJyskEbRERERPSyFeueTe4AyVSoNWocjT6KMuFl4OfqB7m/HCjiXTMzjRqN7l0CAJwuXxMaI566Q60Gjh7NXm/ZEpAb76kQEZGJ0ykgkslkecYImcKYodzSstLQ+cfOSA78t3csCYB1oUUkqqxMhGz8EADgM+k3pOow/qi0SUsDcmZqSUoCrIt4DYiIiEobnQIiIQSGDh0qTeCalpaG999/P89TZlu2bDFcC4mIiIheMJ0CoiFDhmhtv/322wZtDBEREVFJ0CkgWrt27YtqBxEREVGJ0etN1URERESvEgZEREREZPIYEBEREZHJM+65I0qIQq7AvHbzcOifQ2hRoQXMFUW/jFlyOQJbDZPWjZlCASxa9N86ERGRsWJApAelXIkpraYArXQvmylXYFXjtwzeppKgVALTppV0K4iIiIqPt8yIiIjI5LGHSA9qjRpn752F9Z/WqOZcDfIGuk3d4fv3TQDARbfXjH7qjrNns9fr1ePUHUREZLwYEOkhLSsNrb5vpffUHdt/mgzg1Zi6o1Gj7HVO3UFERMaMt8yIiIjI5DEgIiIiIpPHgIiIiIhMHgMiIiIiMnkMiIiIiMjkMSAiIiIik8fH7vWgkCswo9UMHLp/CC0rtIRcUfRH57PkcixrPkBaN2YKBTB79n/rRERExooBkR6UciVmtZsFtNO9bKZcgWUtAgzfqBKgVAJz5pR0K4iIiIqPt8yIiIjI5LGHSA8aoUHU31FQXVOhkmMlmNUwK3JoKRMaVH70FwDghrMnhMx4Y1KNBoiKyl738QHMjPdUiIjIxDEg0kNqZioaLW+k19QdFpkZ2PvjGAA5U3dYvJhGvgSpqYCvb/Y6p+4gIiJjZlT/08+fPx8ymQwTJ06U0oQQmDNnDjw8PGBpaYlWrVrh0qVLWuXS09Mxbtw4ODs7w9raGt26dcO9e/decuuJiIiotDKagCg8PByrVq1CrVq1tNIXLVqEpUuXYsWKFQgPD4e7uzvatWuHp0+fSnkmTpyIrVu3IiQkBGFhYUhKSkKXLl2gVqtf9mkQERFRKWQUAVFSUhICAgLwww8/wMHBQUoXQmDZsmX46KOP0KtXL/j6+mLdunVISUlBcHAwACAhIQFr1qzBkiVL0LZtW9StWxfr16/Hn3/+iX379pXUKREREVEpYhQB0ZgxY9C5c2e0bdtWKz06OhqxsbFo3769lKZSqeDv74/jx48DACIiIpCZmamVx8PDA76+vlIeIiIiMm2lflB1SEgIzp49i/Dw8Dz7YmNjAQBubm5a6W5ubrhz546UR6lUavUs5eTJKZ+f9PR0pKenS9uJiYl6nwMRERGVbqW6h+ivv/7ChAkTsH79elhYFPw0lkwm09oWQuRJy+15eebPnw97e3tp8fT01K3xREREZDRKdUAUERGBuLg41K9fH+bm5jA3N8fhw4fx9ddfw9zcXOoZyt3TExcXJ+1zd3dHRkYG4uPjC8yTn5kzZyIhIUFa/vrrL2mfQq7AuGbjcKj3IagnqwEdpq3IksvxfaNe+L5Rr1di6o6pU7MXTt1BRETGrFTfMmvTpg3+/PNPrbRhw4ahevXq+OCDD1CpUiW4u7tj7969qFu3LgAgIyMDhw8fxsKFCwEA9evXh0KhwN69e9G3b18AQExMDC5evIhFixYVeGyVSgWVSpXvPqVciQWdFwCddT+nTLkC81sP171gKaRUAosXl3QriIiIiq9UB0S2trbwzXnz37+sra3h5OQkpU+cOBGBgYGoUqUKqlSpgsDAQFhZWWHgwIEAAHt7e4wYMQJTpkyBk5MTHB0dMXXqVPj5+eUZpE1ERESmqVQHREUxffp0pKamYvTo0YiPj0fjxo2xZ88e2NraSnm+/PJLmJubo2/fvkhNTUWbNm0QFBQEuZ63rDRCg7vxd2F+zxweth4wq6jb1B3lEh8CAO7buRj91B1372avV6jAqTuIiMh4yYQQoqQbYQwSExNhb2+PhIQEyC3kcJ3rWujUHV4zduZbj2VGGqK+7A2geFN33F6gx/06A0tOBmxssteLOnVHQdfFEErDNSEiotLl2e9vOzu7AvPxf3oiIiIyeQyIiIiIyOQxICIiIiKTx4CIiIiITB4DIiIiIjJ5DIiIiIjI5Bn9e4hKgrmZOd5p+A4Odz6MFp4tIDcv+vuM1GZy/FS3s7RuzMzNgdGj/1snIiIyVvwa04PKXIWvenwF9NC9bIa5Ap+0H2XwNpUElQr45puSbgUREVHx8ZYZERERmTz2EOlBCIFHyY8geySDk5UTZC4yQFbkwnBMTQQAPLG0A2RFLVj6CAE8epS97uxs1KdCREQmjj1EekjJTIHXfC84eztD5iYDUope1jIzHWeXB+Ds8gBYZqa/uEa+BCkpgKtr9pKiwzUgIiIqbRgQERERkcljQEREREQmjwERERERmTwGRERERGTyGBARERGRyWNARERERCaP7yHSg7mZOQbUGYCw1mFoWr6pzlN3/ObbRlo3ZubmwJAh/60TEREZK36N6UFlrsLqPquBPrqXzTBXYGrnSYZvVAlQqYCgoJJuBRERUfHxlhkRERGZPPYQ6UEIgZSMFCAFsFJYQWat29QdOW+oTlWojHq+CyH+e0O1lZVRnwoREZk49hDpISUzBa7zXGHtaA2Zre5Td0R92RtRX/Z+JabusLHJXjh1BxERGTMGRERERGTyGBARERGRyWNARERERCaPARERERGZPAZEREREZPIYEBEREZHJ43uI9CA3k6N7je440eQEGpVrBLm86FNwaMzMsLNac2ndmMnlQO/e/60TEREZKwZEerAwt0BwQDAQoHvZdHMlxvSYafhGlQALC+DXX0u6FURERMVn3F0URERERAbAgIiIiIhMHgMiPSRnJMP6I+vs+ctkAJKLXtYyIw23F3bB7YVdYJmR9qKa+FIkJ2fPXyaTZa8TEREZKwZEREREZPIYEBEREZHJY0BEREREJo8BEREREZk8BkRERERk8hgQERERkcnjm6r1IDeTo12VdgivE456ZevpPHXHgUoNpHVjJpcDnTr9t05ERGSsGBDpwcLcAtuGbgOG6l423VyJ4X3mGLhFJcPCAti5s6RbQUREVHzG3UVBREREZAAMiIiIiMjkMSDSQ3JGMlzmuCBZmQxhLXSeuuPy0rdweelbr8TUHdbW2Qun7iAiImPGMUR6SslMgXWmNZCpe1mrzHTDN6iEpKSUdAuIiIiKjz1EREREZPIYEBEREZHJY0BEREREJo8BEREREZk8BkRERERk8viUmR7MZGZoXrE5IqtFws/ND3IzHabukMlw0tNXWjdmZmaAv/9/60RERMZKJoQQJd2IgsyfPx9btmzBlStXYGlpiWbNmmHhwoWoVq2alEcIgblz52LVqlWIj49H48aN8c0336BmzZpSnvT0dEydOhUbN25Eamoq2rRpg2+//Rbly5cvclsSExNhb2+PhIQE2NnZPTe/1wzjndPi9oLOL6zuF3ldXmS7iYjIOBX1+7tU/19/+PBhjBkzBidPnsTevXuRlZWF9u3bI/mZtwAuWrQIS5cuxYoVKxAeHg53d3e0a9cOT58+lfJMnDgRW7duRUhICMLCwpCUlIQuXbpArVaXxGkRERFRKVOqb5mFhoZqba9duxaurq6IiIjA66+/DiEEli1bho8++gi9evUCAKxbtw5ubm4IDg7GyJEjkZCQgDVr1uDnn39G27ZtAQDr16+Hp6cn9u3bhw4dOrz08yIiIqLSpVT3EOWWkJAAAHB0dAQAREdHIzY2Fu3bt5fyqFQq+Pv74/jx4wCAiIgIZGZmauXx8PCAr6+vlCc/6enpSExM1FpyJGcko+LnFfHI5hGEi+5Td0R8PRARXw98JabucHHJXjh1BxERGTOjCYiEEJg8eTJatGgBX9/sQcmxsbEAADc3N628bm5u0r7Y2FgolUo4ODgUmCc/8+fPh729vbR4enpq7X+U8gjOyc6QPdJ9YLRTaiKcUhOfn9EIPHqUvRARERmzUn3L7Fljx47FhQsXEBYWlmefLNfTWkKIPGm5PS/PzJkzMXnyZGk7MTExT1D0qirqwGdNhhxARwCAz6xQmCk5JouIiIyTUfQQjRs3Dtu3b8fBgwe1ngxzd3cHgDw9PXFxcVKvkbu7OzIyMhAfH19gnvyoVCrY2dlpLURERPRqKtUBkRACY8eOxZYtW3DgwAF4e3tr7ff29oa7uzv27t0rpWVkZODw4cNo1qwZAKB+/fpQKBRaeWJiYnDx4kUpDxEREZm2Un3LbMyYMQgODsb//vc/2NraSj1B9vb2sLS0hEwmw8SJExEYGIgqVaqgSpUqCAwMhJWVFQYOHCjlHTFiBKZMmQInJyc4Ojpi6tSp8PPzk546IyIiItNWqgOilStXAgBatWqllb527VoMHToUADB9+nSkpqZi9OjR0osZ9+zZA1tbWyn/l19+CXNzc/Tt21d6MWNQUBDk8qK/YZqIiIheXaU6ICrKS7RlMhnmzJmDOXPmFJjHwsICy5cvx/Llyw3SLjOZGep61MXlipdR3bk6zHSYt0Ijk+G8exVp3ajJBJTu/0jrRERExqpUT91RmpjS1B3GilN3EBFRbq/E1B1ERERELwMDIiIiIjJ5DIj0kJKZguqLquOe4z1ovDRAStHLWmSmIWzlcIStHA6LTOOeukOTaYZ7K1vj3srW0GTyR4mIiIxXqR5UXVoJIfBXwl8oH18eiAegwygsmQDKJ8ZJ60ZNyKBOtJLWiYiIjBX/rSciIiKTx4CIiIiITB4DIiIiIjJ5DIiIiIjI5DEgIiIiIpPHp8z0IJPJUN2lOm6430Alh0owkxU9rhQy4JpTBWndqMkEFE5PpXUiIiJjxak7iohTd5R+nLqDiIhy49QdREREREXEgIiIiIhMHgMiPaRkpqD+svq4UfYGNDV0n7pjz+rR2LN69CsxdceD1a/jwerXOXUHEREZNQ6q1oMQAlceXkHl2MpALHSeuqPq47vSulETMmQ+tpXWiYiIjBX/rSciIiKTx4CIiIiITB4DIiIiIjJ5DIiIiIjI5DEgIiIiIpPHp8z0IJPJ4GnviXsO9+Bh56Hz1B337FyldaMmE5DbpUjrRERExopTdxQRp+4o/Th1BxER5capO4iIiIiKiAERERERmTwGRHpIzUxFi29a4LLXZWgaaIDUopdVZabjf+sm4X/rJkGVmf7iGvkSaDLNELOuOWLWNefUHUREZNQ4qFoPGqHBuQfnUONODeAOAE3Ry5oJgdqx16V1oyZkyIgtI60TEREZK/5bT0RERCaPARERERGZPAZEREREZPIYEBEREZHJY0BEREREJo9PmenJ2coZj6wfwcnSCTLo9oTVY8vnv+naWJhZGverA4iIiABO3VFknLqj9OPUHURElBun7iAiIiIqIgZEREREZPIYEOkhNTMV7Ve1R2T1SKj91TpP3RESPAMhwTNeiak7YoObIDa4CafuICIio8ZB1XrQCA2O3TmGOlfrAFeh89QdTf66KK0bNSFD+l9O0joREZGx4r/1REREZPIYEBEREZHJY0BEREREJo8BEREREZk8BkRERERk8viUmZ6sFFZIViTDSmGl89QdKQrVC2rVyydTZJV0E4iIiIqNU3cUEafuKP04dQcREeXGqTuIiIiIiogBEREREZk8jiHSQ1pWGvqv74+PvvoI9crWg3yLHLAoWllVVgZWbg0EAIzq+SHSzZUvsKUvlsgyw8Ot9QEALj0jIDPX4ZXdRERkdF7kcJCSHvbAgEgPao0ae6/vxbbIbUAkAHXRy5ppNHjj1hlp3ZgJjQypt1yldU7eQURExooBEb0yXuX/XIiI6MUyqTFE3377Lby9vWFhYYH69evj6NGjJd0kIiIiKgVMJiDatGkTJk6ciI8++gjnzp1Dy5Yt8eabb+Lu3bsl3TQiIiIqYSYTEC1duhQjRozAO++8Ax8fHyxbtgyenp5YuXJlSTeNiIiISphJjCHKyMhAREQEZsyYoZXevn17HD9+vIRaRcaE45OIiF5tJhEQPXr0CGq1Gm5ublrpbm5uiI2NzbdMeno60tPTpe2EhAQA2W+8lGfIIdIFEpGYvTMReZ4006Sn5FuvOiMtpxTU6SnQCON90kyTYQb8ezaa9GTAiM+lJCUmJj4/ExG9Unxn/1HSTSh1XtTfwpx6nzcxh0kERDlkMu0Hw4UQedJyzJ8/H3Pnzs2T7unpKa3bwz57xUO3dtjnrHw7WLeCpdj9b0u6BcbLfllJt4CIqOS96L+FT58+hb29fYH7TSIgcnZ2hlwuz9MbFBcXl6fXKMfMmTMxefJkaVuj0eDJkydwcnKCTCZDYmIiPD098ddffxVpbjPSD6/zy8Hr/HLwOr8cvM4vh7FcZyEEnj59Cg+PwnsvTCIgUiqVqF+/Pvbu3YuePXtK6Xv37kX37t3zLaNSqaBSac9KX6ZMmTz57OzsSvUPwquC1/nl4HV+OXidXw5e55fDGK5zYT1DOUwiIAKAyZMnY9CgQWjQoAGaNm2KVatW4e7du3j//fdLumlERERUwkwmIOrXrx8eP36MefPmISYmBr6+vti1axcqVqxY0k0jIiKiEmYyAREAjB49GqNHjzZIXSqVCrNnz85zW40Mi9f55eB1fjl4nV8OXueX41W7zjLxvOfQiIiIiF5xJvOmaiIiIqKCMCAiIiIik8eAiIiIiEweAyIiIiIyeQyI9PTtt9/C29sbFhYWqF+/Po4ePVrSTTIa8+fPR8OGDWFrawtXV1f06NEDV69e1cojhMCcOXPg4eEBS0tLtGrVCpcuXdLKk56ejnHjxsHZ2RnW1tbo1q0b7t279zJPxajMnz8fMpkMEydOlNJ4nQ3j/v37ePvtt+Hk5AQrKyvUqVMHERER0n5e5+LLysrCxx9/DG9vb1haWqJSpUqYN28eNJr/5lDkddbdkSNH0LVrV3h4eEAmk2Hbtm1a+w11TePj4zFo0CDY29vD3t4egwYNwj///POCz05HgnQWEhIiFAqF+OGHH8Tly5fFhAkThLW1tbhz505JN80odOjQQaxdu1ZcvHhRREZGis6dO4sKFSqIpKQkKc+CBQuEra2t2Lx5s/jzzz9Fv379RNmyZUViYqKU5/333xflypUTe/fuFWfPnhWtW7cWtWvXFllZWSVxWqXa6dOnhZeXl6hVq5aYMGGClM7rXHxPnjwRFStWFEOHDhWnTp0S0dHRYt++feLGjRtSHl7n4vvss8+Ek5OT2LFjh4iOjha//vqrsLGxEcuWLZPy8DrrbteuXeKjjz4SmzdvFgDE1q1btfYb6pp27NhR+Pr6iuPHj4vjx48LX19f0aVLl5d1mkXCgEgPjRo1Eu+//75WWvXq1cWMGTNKqEXGLS4uTgAQhw8fFkIIodFohLu7u1iwYIGUJy0tTdjb24vvvvtOCCHEP//8IxQKhQgJCZHy3L9/X5iZmYnQ0NCXewKl3NOnT0WVKlXE3r17hb+/vxQQ8TobxgcffCBatGhR4H5eZ8Po3LmzGD58uFZar169xNtvvy2E4HU2hNwBkaGu6eXLlwUAcfLkSSnPiRMnBABx5cqVF3xWRcdbZjrKyMhAREQE2rdvr5Xevn17HD9+vIRaZdwSEhIAAI6OjgCA6OhoxMbGal1jlUoFf39/6RpHREQgMzNTK4+Hhwd8fX35OeQyZswYdO7cGW3bttVK53U2jO3bt6NBgwbo06cPXF1dUbduXfzwww/Sfl5nw2jRogX279+Pa9euAQDOnz+PsLAwdOrUCQCv84tgqGt64sQJ2Nvbo3HjxlKeJk2awN7evlRdd5N6U7UhPHr0CGq1Gm5ublrpbm5uiI2NLaFWGS8hBCZPnowWLVrA19cXAKTrmN81vnPnjpRHqVTCwcEhTx5+Dv8JCQnB2bNnER4enmcfr7Nh3Lp1CytXrsTkyZPx4Ycf4vTp0xg/fjxUKhUGDx7M62wgH3zwARISElC9enXI5XKo1Wp8/vnnGDBgAAD+PL8IhrqmsbGxcHV1zVO/q6trqbruDIj0JJPJtLaFEHnS6PnGjh2LCxcuICwsLM8+fa4xP4f//PXXX5gwYQL27NkDCwuLAvPxOhePRqNBgwYNEBgYCACoW7cuLl26hJUrV2Lw4MFSPl7n4tm0aRPWr1+P4OBg1KxZE5GRkZg4cSI8PDwwZMgQKR+vs+EZ4prml7+0XXfeMtORs7Mz5HJ5nqg2Li4uTxRNhRs3bhy2b9+OgwcPonz58lK6u7s7ABR6jd3d3ZGRkYH4+PgC85i6iIgIxMXFoX79+jA3N4e5uTkOHz6Mr7/+Gubm5tJ14nUunrJly6JGjRpaaT4+Prh79y4A/jwbyrRp0zBjxgz0798ffn5+GDRoECZNmoT58+cD4HV+EQx1Td3d3fH333/nqf/hw4el6rozINKRUqlE/fr1sXfvXq30vXv3olmzZiXUKuMihMDYsWOxZcsWHDhwAN7e3lr7vb294e7urnWNMzIycPjwYeka169fHwqFQitPTEwMLl68yM/hX23atMGff/6JyMhIaWnQoAECAgIQGRmJSpUq8TobQPPmzfO8NuLatWuoWLEiAP48G0pKSgrMzLS/suRyufTYPa+z4RnqmjZt2hQJCQk4ffq0lOfUqVNISEgoXde9JEZyG7ucx+7XrFkjLl++LCZOnCisra3F7du3S7ppRmHUqFHC3t5eHDp0SMTExEhLSkqKlGfBggXC3t5ebNmyRfz5559iwIAB+T7qWb58ebFv3z5x9uxZ8cYbb5j047NF8exTZkLwOhvC6dOnhbm5ufj888/F9evXxYYNG4SVlZVYv369lIfXufiGDBkiypUrJz12v2XLFuHs7CymT58u5eF11t3Tp0/FuXPnxLlz5wQAsXTpUnHu3DnpNTKGuqYdO3YUtWrVEidOnBAnTpwQfn5+fOz+VfHNN9+IihUrCqVSKerVqyc9Mk7PByDfZe3atVIejUYjZs+eLdzd3YVKpRKvv/66+PPPP7XqSU1NFWPHjhWOjo7C0tJSdOnSRdy9e/cln41xyR0Q8Tobxu+//y58fX2FSqUS1atXF6tWrdLaz+tcfImJiWLChAmiQoUKwsLCQlSqVEl89NFHIj09XcrD66y7gwcP5vv3eMiQIUIIw13Tx48fi4CAAGFraytsbW1FQECAiI+Pf0lnWTQyIYQomb4pIiIiotKBY4iIiIjI5DEgIiIiIpPHgIiIiIhMHgMiIiIiMnkMiIiIiMjkMSAiIiIik8eAiIiIiEweAyIiMjiZTIZt27aVdDNKvVmzZuG9994zeL1xcXFwcXHB/fv3DV430auKARERFcnQoUMhk8kgk8mgUCjg5uaGdu3a4ccff5Tmk8oRExODN998s4Raahz+/vtvfPXVV/jwww+10mNjYzFu3DhUqlQJKpUKnp6e6Nq1K/bv31/kul1dXTFo0CDMnj3b0M0memUxICKiIuvYsSNiYmJw+/Zt7N69G61bt8aECRPQpUsXZGVlSfnc3d2hUqleWDvUanWeIMzYrFmzBk2bNoWXl5eUdvv2bdSvXx8HDhzAokWL8OeffyI0NBStW7fGmDFjdKp/2LBh2LBhQ55ZyIkofwyIiKjIVCoV3N3dUa5cOdSrVw8ffvgh/ve//2H37t0ICgqS8j17y6xp06aYMWOGVj0PHz6EQqHAwYMHAWTPoD19+nSUK1cO1tbWaNy4MQ4dOiTlDwoKQpkyZbBjxw7UqFEDKpUKd+7cQUxMDDp37gxLS0t4e3sjODgYXl5eWLZsmVQ2ISEB7733HlxdXWFnZ4c33ngD58+fl/bPmTMHderUwc8//wwvLy/Y29ujf//+ePr0qZRHo9Fg4cKFqFy5MlQqFSpUqIDPP/9c2n///n3069cPDg4OcHJyQvfu3XH79u1Cr2VISAi6deumlTZ69GjIZDKcPn0avXv3RtWqVVGzZk1MnjwZJ0+e1Lq+K1euxJtvvimd+6+//qpVl5+fH9zd3bF169ZC20FE2RgQEVGxvPHGG6hduza2bNmS7/6AgABs3LgRz06buGnTJri5ucHf3x9Adm/GsWPHEBISggsXLqBPnz7o2LEjrl+/LpVJSUnB/PnzsXr1aly6dAmurq4YPHgwHjx4gEOHDmHz5s1YtWoV4uLipDJCCHTu3BmxsbHYtWsXIiIiUK9ePbRp0wZPnjyR8t28eRPbtm3Djh07sGPHDhw+fBgLFiyQ9s+cORMLFy7ErFmzcPnyZQQHB8PNzU1qV+vWrWFjY4MjR44gLCwMNjY26NixIzIyMvK9JvHx8bh48SIaNGggpT158gShoaEYM2YMrK2t85QpU6aM1vasWbPw1ltv4fz583j77bcxYMAAREVFaeVp1KgRjh49mm8biCiXkp1bloiMxZAhQ0T37t3z3devXz/h4+MjbQMQW7duFUIIERcXJ8zNzcWRI0ek/U2bNhXTpk0TQghx48YNIZPJxP3797XqbNOmjZg5c6YQQoi1a9cKACIyMlLaHxUVJQCI8PBwKe369esCgPjyyy+FEELs379f2NnZibS0NK26X3vtNfH9998LIYSYPXu2sLKyEomJidL+adOmicaNGwshsmdZV6lU4ocffsj33NesWSOqVasmNBqNlJaeni4sLS3FH3/8kW+Zc+fOCQBaM4KfOnVKABBbtmzJt8yzAIj3339fK61x48Zi1KhRWmmTJk0SrVq1em59RCSEeQnGYkT0ihBCQCaT5bvPxcUF7dq1w4YNG9CyZUtER0fjxIkTWLlyJQDg7NmzEEKgatWqWuXS09Ph5OQkbSuVStSqVUvavnr1KszNzVGvXj0prXLlynBwcJC2IyIikJSUpFUPAKSmpuLmzZvStpeXF2xtbaXtsmXLSj1NUVFRSE9PR5s2bfI9v4iICNy4cUOrPACkpaVpHSP38QHAwsJCShP/9qAVdB1za9q0aZ7tyMhIrTRLS0ukpKQUqT4iU8eAiIiKLSoqCt7e3gXuDwgIwIQJE7B8+XIEBwejZs2aqF27NoDs8TlyuRwRERGQy+Va5WxsbKR1S0tLrWBBPHML7lnPpms0GpQtW1ZrPFKOZ29BKRQKrX0ymUwatG1paVngeeUco379+tiwYUOefS4uLvmWcXZ2BpB96ywnT5UqVSCTyRAVFYUePXoUesyC5A6mnjx5UmAbiEgbxxARUbEcOHAAf/75J956660C8/To0QNpaWkIDQ1FcHAw3n77bWlf3bp1oVarERcXh8qVK2st7u7uBdZZvXp1ZGVl4dy5c1LajRs38M8//0jb9erVQ2xsLMzNzfPUnROUPE+VKlVgaWlZ4GPv9erVw/Xr1+Hq6prnGPb29vmWee2112BnZ4fLly9LaY6OjujQoQO++eYbJCcn5ynz7HkB0BpknbNdvXp1rbSLFy+ibt26RTlNIpPHgIiIiiw9PR2xsbG4f/8+zp49i8DAQHTv3h1dunTB4MGDCyxnbW2N7t27Y9asWYiKisLAgQOlfVWrVkVAQAAGDx6MLVu2IDo6GuHh4Vi4cCF27dpVYJ3Vq1dH27Zt8d577+H06dM4d+4c3nvvPa2epLZt26Jp06bo0aMH/vjjD9y+fRvHjx/Hxx9/jDNnzhTpnC0sLPDBBx9g+vTp+Omnn3Dz5k2cPHkSa9asAZDd++Xs7Izu3bvj6NGjiI6OxuHDhzFhwgTcu3cv3zrNzMzQtm1bhIWFaaV/++23UKvVaNSoETZv3ozr168jKioKX3/9dZ5bZL/++it+/PFHXLt2DbNnz8bp06cxduxYaX9KSgoiIiLQvn37Ip0nkckr0RFMRGQ0hgwZIgAIAMLc3Fy4uLiItm3bih9//FGo1WqtvHhmUHWOnTt3CgDi9ddfz1N3RkaG+OSTT4SXl5dQKBTC3d1d9OzZU1y4cEEIkT2o2t7ePk+5Bw8eiDfffFOoVCpRsWJFERwcLFxdXcV3330n5UlMTBTjxo0THh4eQqFQCE9PTxEQECANaJ49e7aoXbu2Vr1ffvmlqFixorStVqvFZ599JipWrCgUCoWoUKGCCAwMlPbHxMSIwYMHC2dnZ6FSqUSlSpXEu+++KxISEgq8nqGhoaJcuXJ5rt2DBw/EmDFjRMWKFYVSqRTlypUT3bp1EwcPHtS6vt98841o166ddO4bN27Uqic4OFhUq1atwOMTkTaZEAXciCciMjL37t2Dp6cn9u3bV+Ag6NJCCIEmTZpg4sSJGDBggE5lZTIZtm7dWuhYo0aNGmHixIlavXFEVDAOqiYio3XgwAEkJSXBz88PMTExmD59Ory8vPD666+XdNOeSyaTYdWqVbhw4YLB646Li0Pv3r11DrSITBkDIiIyWpmZmfjwww9x69Yt2NraolmzZtiwYUOep8ZKq9q1a0tP2xmSq6srpk+fbvB6iV5lvGVGREREJo9PmREREZHJY0BEREREJo8BEREREZk8BkRERERk8hgQERERkcljQEREREQmjwERERERmTwGRERERGTyGBARERGRyfs/Yt8kUh7b1JwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "generate_frequency_graph(new_m16, 'avg_tactic_ground_avg', \"Divergence $(T,$ Maia-1600, $P_{test})$\", \"Divergence (Cp)\", left=-10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "\n",
    "matplotlib.use(\"pgf\")\n",
    "matplotlib.rcParams.update({\n",
    "    \"pgf.texsystem\": \"pdflatex\",\n",
    "    'font.family': 'serif',\n",
    "    'text.usetex': True,\n",
    "    'pgf.rcfonts': False,\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame({\n",
    "    \"old\": old_m16['avg_tactic_ground_avg'],\n",
    "    \"none\": none_m16['avg_tactic_ground_avg'],\n",
    "    \"prec\": prec_m16['avg_tactic_ground_avg'],\n",
    "    \"rec\": rec_m16['avg_tactic_ground_avg'],\n",
    "    \"new\": new_m16['avg_tactic_ground_avg'],\n",
    "})\n",
    "\n",
    "rand_line = plt.axvline(random_tactic['avg_tactic_ground_avg'].values, linestyle='dashed', color='red', label='random move tactic', linewidth=1)\n",
    "ground_line = plt.axvline(ground_tactic['avg_tactic_ground_avg'].values, linestyle='dashed', color='green', label='ground move tactic', linewidth=1)\n",
    "m1600_line = plt.axvline(m1600_tactic['avg_tactic_ground_avg'].values, linestyle='dashed', color='blue', label='Maia-1600 move tactic', linewidth=1)\n",
    "sf14_line = plt.axvline(sf14_tactic['avg_tactic_ground_avg'].values, linestyle='dashed', color='magenta', label='Stockfish 14 move tactic', linewidth=1)\n",
    "\n",
    "# ax = data[[\"old\", \"none\", \"prec\", \"rec\", \"new\"]].plot(kind='box', title='Box Plot of Divergences (Maia-1600) of Tactics Learned', vert=False)\n",
    "sns.boxplot(y=\"variable\", x=\"value\", data=pd.melt(data[[\"old\", \"none\", \"prec\", \"rec\", \"new\"]]), palette='pastel')\n",
    "plt.xlabel('Divergence (Cp)')\n",
    "plt.ylabel('Tactic Sets')\n",
    "plt.title('Boxplot of Divergence (Maia-1600)')\n",
    "plt.legend(loc=1)\n",
    "plt.savefig('boxplot_m1600.pgf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame({\n",
    "    \"old\": old_sf14['avg_tactic_ground_avg'],\n",
    "    \"none\": none_sf14['avg_tactic_ground_avg'],\n",
    "    \"prec\": prec_sf14['avg_tactic_ground_avg'],\n",
    "    \"rec\": rec_sf14['avg_tactic_ground_avg'],\n",
    "    \"new\": new_sf14['avg_tactic_ground_avg'],\n",
    "})\n",
    "\n",
    "rand_line = plt.axvline(random_tactic['avg_tactic_ground_avg'].values, linestyle='dashed', color='red', label='random move tactic', linewidth=1)\n",
    "ground_line = plt.axvline(ground_tactic['avg_tactic_ground_avg'].values, linestyle='dashed', color='green', label='ground move tactic', linewidth=1)\n",
    "m1600_line = plt.axvline(m1600_tactic['avg_tactic_ground_avg'].values, linestyle='dashed', color='blue', label='Maia-1600 move tactic', linewidth=1)\n",
    "sf14_line = plt.axvline(sf14_tactic['avg_tactic_ground_avg'].values, linestyle='dashed', color='magenta', label='Stockfish 14 move tactic', linewidth=1)\n",
    "\n",
    "# ax = data[[\"old\", \"none\", \"prec\", \"rec\", \"new\"]].plot(kind='box', title='Box Plot of Divergences (Maia-1600) of Tactics Learned', vert=False)\n",
    "sns.boxplot(y=\"variable\", x=\"value\", data=pd.melt(data[[\"old\", \"none\", \"prec\", \"rec\", \"new\"]]), palette='pastel')\n",
    "plt.xlabel('Divergence (Cp)')\n",
    "plt.ylabel('Tactic Sets')\n",
    "plt.title('Boxplot of Divergence (Stockfish 14)')\n",
    "plt.legend(loc=1)\n",
    "plt.savefig('boxplot_sf14.pgf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==old m16==\n",
      "match                      4.461356\n",
      "num_suggestions           41.369798\n",
      "correct_move               0.661118\n",
      "tactic_ground_div        468.522711\n",
      "tactic_ground_avg        732.285322\n",
      "avg_tactic_ground_avg    174.484153\n",
      "coverage                   0.446136\n",
      "accuracy                   0.120367\n",
      "dtype: float64 match                      1.586773\n",
      "num_suggestions           50.462433\n",
      "correct_move               0.863560\n",
      "tactic_ground_div        137.104788\n",
      "tactic_ground_avg        262.169046\n",
      "avg_tactic_ground_avg     48.693434\n",
      "coverage                   0.158677\n",
      "accuracy                   0.130892\n",
      "dtype: float64\n",
      "==none m16==\n",
      "match                      26.372414\n",
      "num_suggestions            26.372414\n",
      "correct_move                5.751724\n",
      "tactic_ground_div        2936.510345\n",
      "tactic_ground_avg        2936.510345\n",
      "avg_tactic_ground_avg     103.278407\n",
      "coverage                    0.263724\n",
      "accuracy                    0.336175\n",
      "dtype: float64 match                      35.004155\n",
      "num_suggestions            35.004155\n",
      "correct_move               10.371255\n",
      "tactic_ground_div        4466.515648\n",
      "tactic_ground_avg        4466.515648\n",
      "avg_tactic_ground_avg      63.666757\n",
      "coverage                    0.350042\n",
      "accuracy                    0.235854\n",
      "dtype: float64\n",
      "==prec m16\n",
      "match                      20.699029\n",
      "num_suggestions            20.310680\n",
      "correct_move                4.203883\n",
      "tactic_ground_div        7442.582524\n",
      "tactic_ground_avg        7442.582524\n",
      "avg_tactic_ground_avg     276.587045\n",
      "coverage                    0.206990\n",
      "accuracy                    0.317959\n",
      "dtype: float64 match                       29.298954\n",
      "num_suggestions             28.904589\n",
      "correct_move                 5.269321\n",
      "tactic_ground_div        13087.349685\n",
      "tactic_ground_avg        13087.349685\n",
      "avg_tactic_ground_avg      147.126566\n",
      "coverage                     0.292990\n",
      "accuracy                     0.226651\n",
      "dtype: float64\n",
      "==rec m16==\n",
      "match                      51.956522\n",
      "num_suggestions            51.956522\n",
      "correct_move                5.150895\n",
      "tactic_ground_div        6871.222506\n",
      "tactic_ground_avg        6871.222506\n",
      "avg_tactic_ground_avg     143.272511\n",
      "coverage                    0.519565\n",
      "accuracy                    0.103937\n",
      "dtype: float64 match                      40.570175\n",
      "num_suggestions            40.570175\n",
      "correct_move                7.136417\n",
      "tactic_ground_div        8559.291308\n",
      "tactic_ground_avg        8559.291308\n",
      "avg_tactic_ground_avg     161.611803\n",
      "coverage                    0.405702\n",
      "accuracy                    0.121744\n",
      "dtype: float64\n",
      "==new m16==\n",
      "match                      53.739583\n",
      "num_suggestions            53.739583\n",
      "correct_move                5.680556\n",
      "tactic_ground_div        7126.246528\n",
      "tactic_ground_avg        7126.246528\n",
      "avg_tactic_ground_avg     143.568763\n",
      "coverage                    0.537396\n",
      "accuracy                    0.109310\n",
      "dtype: float64 match                      40.536965\n",
      "num_suggestions            40.536965\n",
      "correct_move                8.049631\n",
      "tactic_ground_div        9291.815386\n",
      "tactic_ground_avg        9291.815386\n",
      "avg_tactic_ground_avg     167.533309\n",
      "coverage                    0.405370\n",
      "accuracy                    0.125428\n",
      "dtype: float64\n",
      "==old sf14==\n",
      "match                       4.461356\n",
      "num_suggestions            41.369798\n",
      "correct_move                0.659929\n",
      "tactic_ground_div         913.120047\n",
      "tactic_ground_avg        1496.396215\n",
      "avg_tactic_ground_avg     337.564711\n",
      "coverage                    0.446136\n",
      "accuracy                    0.120248\n",
      "dtype: float64 match                      1.586773\n",
      "num_suggestions           50.462433\n",
      "correct_move               0.863782\n",
      "tactic_ground_div        336.013203\n",
      "tactic_ground_avg        614.192437\n",
      "avg_tactic_ground_avg     57.982444\n",
      "coverage                   0.158677\n",
      "accuracy                   0.130956\n",
      "dtype: float64\n",
      "==none sf14==\n",
      "match                      26.372414\n",
      "num_suggestions            24.993103\n",
      "correct_move                4.993103\n",
      "tactic_ground_div        9214.337931\n",
      "tactic_ground_avg        9214.337931\n",
      "avg_tactic_ground_avg     268.448936\n",
      "coverage                    0.263724\n",
      "accuracy                    0.327582\n",
      "dtype: float64 match                       35.004155\n",
      "num_suggestions             34.025623\n",
      "correct_move                 8.536276\n",
      "tactic_ground_div        15158.006283\n",
      "tactic_ground_avg        15158.006283\n",
      "avg_tactic_ground_avg      131.671913\n",
      "coverage                     0.350042\n",
      "accuracy                     0.237722\n",
      "dtype: float64\n",
      "==prec sf1==4\n",
      "match                      20.699029\n",
      "num_suggestions            20.310680\n",
      "correct_move                4.203883\n",
      "tactic_ground_div        7442.582524\n",
      "tactic_ground_avg        7442.582524\n",
      "avg_tactic_ground_avg     276.587045\n",
      "coverage                    0.206990\n",
      "accuracy                    0.317959\n",
      "dtype: float64 match                       29.298954\n",
      "num_suggestions             28.904589\n",
      "correct_move                 5.269321\n",
      "tactic_ground_div        13087.349685\n",
      "tactic_ground_avg        13087.349685\n",
      "avg_tactic_ground_avg      147.126566\n",
      "coverage                     0.292990\n",
      "accuracy                     0.226651\n",
      "dtype: float64\n",
      "==rec sf14==\n",
      "match                       51.961637\n",
      "num_suggestions             51.450128\n",
      "correct_move                 4.869565\n",
      "tactic_ground_div        22893.046036\n",
      "tactic_ground_avg        22893.046036\n",
      "avg_tactic_ground_avg      467.788013\n",
      "coverage                     0.519616\n",
      "accuracy                     0.100862\n",
      "dtype: float64 match                       40.573088\n",
      "num_suggestions             40.594643\n",
      "correct_move                 6.157411\n",
      "tactic_ground_div        18213.125619\n",
      "tactic_ground_avg        18213.125619\n",
      "avg_tactic_ground_avg      138.685928\n",
      "coverage                     0.405731\n",
      "accuracy                     0.117143\n",
      "dtype: float64\n",
      "==new sf14==\n",
      "match                       53.739583\n",
      "num_suggestions             53.045139\n",
      "correct_move                 5.298611\n",
      "tactic_ground_div        23832.777778\n",
      "tactic_ground_avg        23832.777778\n",
      "avg_tactic_ground_avg      473.312884\n",
      "coverage                     0.537396\n",
      "accuracy                     0.105127\n",
      "dtype: float64 match                       40.536965\n",
      "num_suggestions             40.595241\n",
      "correct_move                 6.890462\n",
      "tactic_ground_div        18583.179704\n",
      "tactic_ground_avg        18583.179704\n",
      "avg_tactic_ground_avg      151.899036\n",
      "coverage                     0.405370\n",
      "accuracy                     0.119476\n",
      "dtype: float64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_14675/1207209941.py:2: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  print(old_m16.mean(), old_m16.std())\n",
      "/tmp/ipykernel_14675/1207209941.py:5: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  print(none_m16.mean(), none_m16.std())\n",
      "/tmp/ipykernel_14675/1207209941.py:8: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  print(prec_m16.mean(), prec_m16.std())\n",
      "/tmp/ipykernel_14675/1207209941.py:11: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  print(rec_m16.mean(), rec_m16.std())\n",
      "/tmp/ipykernel_14675/1207209941.py:14: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  print(new_m16.mean(), new_m16.std())\n",
      "/tmp/ipykernel_14675/1207209941.py:17: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  print(old_sf14.mean(), old_sf14.std())\n",
      "/tmp/ipykernel_14675/1207209941.py:20: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  print(none_sf14.mean(), none_sf14.std())\n",
      "/tmp/ipykernel_14675/1207209941.py:23: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  print(prec_sf14.mean(), prec_sf14.std())\n",
      "/tmp/ipykernel_14675/1207209941.py:26: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  print(rec_sf14.mean(), rec_sf14.std())\n",
      "/tmp/ipykernel_14675/1207209941.py:29: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  print(new_sf14.mean(), new_sf14.std())\n"
     ]
    }
   ],
   "source": [
    "print('==old m16==')\n",
    "print(old_m16.mean(), old_m16.std())\n",
    "\n",
    "print('==none m16==')\n",
    "print(none_m16.mean(), none_m16.std())\n",
    "\n",
    "print('==prec m16')\n",
    "print(prec_m16.mean(), prec_m16.std())\n",
    "\n",
    "print('==rec m16==')\n",
    "print(rec_m16.mean(), rec_m16.std())\n",
    "\n",
    "print('==new m16==')\n",
    "print(new_m16.mean(), new_m16.std())\n",
    "\n",
    "print('==old sf14==')\n",
    "print(old_sf14.mean(), old_sf14.std())\n",
    "\n",
    "print('==none sf14==')\n",
    "print(none_sf14.mean(), none_sf14.std())\n",
    "\n",
    "print('==prec sf1==4')\n",
    "print(prec_sf14.mean(), prec_sf14.std())\n",
    "\n",
    "print('==rec sf14==')\n",
    "print(rec_sf14.mean(), rec_sf14.std())\n",
    "\n",
    "print('==new sf14==')\n",
    "print(new_sf14.mean(), new_sf14.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ttest_indResult(statistic=-14.17362970605678, pvalue=1.0)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sp.stats.ttest_ind(old_sf14['avg_tactic_ground_avg'], new_sf14['avg_tactic_ground_avg'], equal_var=False, alternative='greater', random_state=1, nan_policy='omit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.width', 1000)\n",
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tactic_text</th>\n",
       "      <th>match</th>\n",
       "      <th>num_suggestions</th>\n",
       "      <th>correct_move</th>\n",
       "      <th>tactic_ground_div</th>\n",
       "      <th>tactic_ground_avg</th>\n",
       "      <th>avg_tactic_ground_avg</th>\n",
       "      <th>coverage</th>\n",
       "      <th>accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>f(A,B):-legal_move(A,B),kingside_castle(A,C),k...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>f(A,B):-en_passant(A,D),kingside_castle(A,C),l...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>f(A,B):-legal_move(A,B),turn(A,C),kingside_cas...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>f(A,B):-en_passant(A,C),kingside_castle(A,E),l...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>f(A,B):-legal_move(A,B),en_passant(A,D),remove...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>f(A,B):-kingside_castle(A,C),legal_move(A,B),l...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>f(A,B):-legal_move(A,B),en_passant(A,D),remove...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>f(A,B):-legal_move(A,B),kingside_castle(A,D),l...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>f(A,B):-legal_move(A,B),kingside_castle(A,D),e...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>f(A,B):-legal_move(A,B),kingside_castle(A,C),t...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>f(A,B):-en_passant(A,D),legal_move(A,B),make_m...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>f(A,B):-legal_move(A,B),kingside_castle(A,C),o...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>f(A,B):-legal_move(A,B),kingside_castle(A,C)</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>f(A,B):-kingside_castle(A,C),legal_move(A,B),m...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>f(A,B):-legal_move(A,B),make_move(A,B,C),kings...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>f(A,B):-kingside_castle(A,C),legal_move(A,B),e...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>f(A,B):-kingside_castle(A,C),en_passant(A,E),l...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>f(A,B):-legal_move(A,B),kingside_castle(A,C),c...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>62.103448</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.103448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>f(A,B):-en_passant(A,E),legal_move(A,B),queens...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234</th>\n",
       "      <td>f(A,B):-legal_move(A,B),queenside_castle(A,D),...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166</th>\n",
       "      <td>f(A,B):-legal_move(A,B),make_move(A,B,C),queen...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2322.0</td>\n",
       "      <td>2322.0</td>\n",
       "      <td>74.903226</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>f(A,B):-legal_move(A,B),make_move(A,B,C),queen...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2322.0</td>\n",
       "      <td>2322.0</td>\n",
       "      <td>74.903226</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>f(A,B):-legal_move(A,B),make_move(A,B,C),queen...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>f(A,B):-legal_move(A,B),make_move(A,B,D),queen...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>f(A,B):-legal_move(A,B),make_move(A,B,D),queen...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2322.0</td>\n",
       "      <td>2322.0</td>\n",
       "      <td>74.903226</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>f(A,B):-en_passant(A,D),remove_piece_at(A,D,C)...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>273</th>\n",
       "      <td>f(A,B):-queenside_castle(A,E),legal_move(A,B),...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>f(A,B):-en_passant(A,C),legal_move(A,B),queens...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272</th>\n",
       "      <td>f(A,B):-queenside_castle(A,E),en_passant(A,D),...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>271</th>\n",
       "      <td>f(A,B):-queenside_castle(A,D),legal_move(A,C),...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>f(A,B):-legal_move(A,B),en_passant(A,D),queens...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>f(A,B):-legal_move(A,B),queenside_castle(A,C),...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270</th>\n",
       "      <td>f(A,B):-queenside_castle(A,D),legal_move(A,B),...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>f(A,B):-legal_move(A,B),queenside_castle(A,D),...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>f(A,B):-legal_move(A,B),queenside_castle(A,C)</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269</th>\n",
       "      <td>f(A,B):-queenside_castle(A,C),turn(A,D),legal_...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>f(A,B):-en_passant(A,D),remove_piece_at(A,D,E)...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>f(A,B):-queenside_castle(A,C),legal_move(A,B),...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268</th>\n",
       "      <td>f(A,B):-queenside_castle(A,C),legal_move(A,B),...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>f(A,B):-en_passant(A,D),remove_piece_at(A,D,E)...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>f(A,B):-legal_move(A,B),en_passant(A,C),queens...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>2217.0</td>\n",
       "      <td>71.516129</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>286</th>\n",
       "      <td>random</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>13324.0</td>\n",
       "      <td>13324.0</td>\n",
       "      <td>133.240000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.090000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251</th>\n",
       "      <td>f(A,B):-legal_move(A,C),pawn_capture(A,C),lega...</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2528.0</td>\n",
       "      <td>2528.0</td>\n",
       "      <td>210.666667</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>f(A,B):-legal_move(A,C),pawn_capture(A,C),lega...</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2528.0</td>\n",
       "      <td>2528.0</td>\n",
       "      <td>210.666667</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>f(A,B):-legal_move(A,B),make_move(A,B,D),turn(...</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>13201.0</td>\n",
       "      <td>13201.0</td>\n",
       "      <td>132.010000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.080000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>f(A,B):-legal_move(A,B),make_move(A,B,D),legal...</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>13201.0</td>\n",
       "      <td>13201.0</td>\n",
       "      <td>132.010000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.080000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>f(A,B):-en_passant(A,D),legal_move(A,B),remove...</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>13201.0</td>\n",
       "      <td>13201.0</td>\n",
       "      <td>132.010000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.080000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>f(A,B):-legal_move(A,B),move(B,D,C,E),differen...</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>13201.0</td>\n",
       "      <td>13201.0</td>\n",
       "      <td>132.010000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.080000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>f(A,B):-en_passant(A,D),legal_move(A,B),en_pas...</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>13201.0</td>\n",
       "      <td>13201.0</td>\n",
       "      <td>132.010000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.080000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>f(A,B):-legal_move(A,B),move(B,C,D,E),differen...</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>13201.0</td>\n",
       "      <td>13201.0</td>\n",
       "      <td>132.010000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.080000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           tactic_text  match  \\\n",
       "130  f(A,B):-legal_move(A,B),kingside_castle(A,C),k...   29.0   \n",
       "26   f(A,B):-en_passant(A,D),kingside_castle(A,C),l...   29.0   \n",
       "236  f(A,B):-legal_move(A,B),turn(A,C),kingside_cas...   29.0   \n",
       "12   f(A,B):-en_passant(A,C),kingside_castle(A,E),l...   29.0   \n",
       "108  f(A,B):-legal_move(A,B),en_passant(A,D),remove...   29.0   \n",
       "75   f(A,B):-kingside_castle(A,C),legal_move(A,B),l...   29.0   \n",
       "117  f(A,B):-legal_move(A,B),en_passant(A,D),remove...   29.0   \n",
       "135  f(A,B):-legal_move(A,B),kingside_castle(A,D),l...   29.0   \n",
       "134  f(A,B):-legal_move(A,B),kingside_castle(A,D),e...   29.0   \n",
       "133  f(A,B):-legal_move(A,B),kingside_castle(A,C),t...   29.0   \n",
       "33   f(A,B):-en_passant(A,D),legal_move(A,B),make_m...   29.0   \n",
       "131  f(A,B):-legal_move(A,B),kingside_castle(A,C),o...   29.0   \n",
       "128       f(A,B):-legal_move(A,B),kingside_castle(A,C)   29.0   \n",
       "76   f(A,B):-kingside_castle(A,C),legal_move(A,B),m...   29.0   \n",
       "153  f(A,B):-legal_move(A,B),make_move(A,B,C),kings...   29.0   \n",
       "74   f(A,B):-kingside_castle(A,C),legal_move(A,B),e...   29.0   \n",
       "73   f(A,B):-kingside_castle(A,C),en_passant(A,E),l...   29.0   \n",
       "129  f(A,B):-legal_move(A,B),kingside_castle(A,C),c...   29.0   \n",
       "71   f(A,B):-en_passant(A,E),legal_move(A,B),queens...   31.0   \n",
       "234  f(A,B):-legal_move(A,B),queenside_castle(A,D),...   31.0   \n",
       "166  f(A,B):-legal_move(A,B),make_move(A,B,C),queen...   31.0   \n",
       "165  f(A,B):-legal_move(A,B),make_move(A,B,C),queen...   31.0   \n",
       "164  f(A,B):-legal_move(A,B),make_move(A,B,C),queen...   31.0   \n",
       "175  f(A,B):-legal_move(A,B),make_move(A,B,D),queen...   31.0   \n",
       "176  f(A,B):-legal_move(A,B),make_move(A,B,D),queen...   31.0   \n",
       "57   f(A,B):-en_passant(A,D),remove_piece_at(A,D,C)...   31.0   \n",
       "273  f(A,B):-queenside_castle(A,E),legal_move(A,B),...   31.0   \n",
       "19   f(A,B):-en_passant(A,C),legal_move(A,B),queens...   31.0   \n",
       "272  f(A,B):-queenside_castle(A,E),en_passant(A,D),...   31.0   \n",
       "271  f(A,B):-queenside_castle(A,D),legal_move(A,C),...   31.0   \n",
       "104  f(A,B):-legal_move(A,B),en_passant(A,D),queens...   31.0   \n",
       "231  f(A,B):-legal_move(A,B),queenside_castle(A,C),...   31.0   \n",
       "270  f(A,B):-queenside_castle(A,D),legal_move(A,B),...   31.0   \n",
       "233  f(A,B):-legal_move(A,B),queenside_castle(A,D),...   31.0   \n",
       "230      f(A,B):-legal_move(A,B),queenside_castle(A,C)   31.0   \n",
       "269  f(A,B):-queenside_castle(A,C),turn(A,D),legal_...   31.0   \n",
       "65   f(A,B):-en_passant(A,D),remove_piece_at(A,D,E)...   31.0   \n",
       "267  f(A,B):-queenside_castle(A,C),legal_move(A,B),...   31.0   \n",
       "268  f(A,B):-queenside_castle(A,C),legal_move(A,B),...   31.0   \n",
       "66   f(A,B):-en_passant(A,D),remove_piece_at(A,D,E)...   31.0   \n",
       "94   f(A,B):-legal_move(A,B),en_passant(A,C),queens...   31.0   \n",
       "286                                             random  100.0   \n",
       "251  f(A,B):-legal_move(A,C),pawn_capture(A,C),lega...   12.0   \n",
       "250  f(A,B):-legal_move(A,C),pawn_capture(A,C),lega...   12.0   \n",
       "177  f(A,B):-legal_move(A,B),make_move(A,B,D),turn(...  100.0   \n",
       "172  f(A,B):-legal_move(A,B),make_move(A,B,D),legal...  100.0   \n",
       "45   f(A,B):-en_passant(A,D),legal_move(A,B),remove...  100.0   \n",
       "198  f(A,B):-legal_move(A,B),move(B,D,C,E),differen...  100.0   \n",
       "27   f(A,B):-en_passant(A,D),legal_move(A,B),en_pas...  100.0   \n",
       "183  f(A,B):-legal_move(A,B),move(B,C,D,E),differen...  100.0   \n",
       "\n",
       "     num_suggestions  correct_move  tactic_ground_div  tactic_ground_avg  \\\n",
       "130             29.0           3.0             1801.0             1801.0   \n",
       "26              29.0           3.0             1801.0             1801.0   \n",
       "236             29.0           3.0             1801.0             1801.0   \n",
       "12              29.0           3.0             1801.0             1801.0   \n",
       "108             29.0           3.0             1801.0             1801.0   \n",
       "75              29.0           3.0             1801.0             1801.0   \n",
       "117             29.0           3.0             1801.0             1801.0   \n",
       "135             29.0           3.0             1801.0             1801.0   \n",
       "134             29.0           3.0             1801.0             1801.0   \n",
       "133             29.0           3.0             1801.0             1801.0   \n",
       "33              29.0           3.0             1801.0             1801.0   \n",
       "131             29.0           3.0             1801.0             1801.0   \n",
       "128             29.0           3.0             1801.0             1801.0   \n",
       "76              29.0           3.0             1801.0             1801.0   \n",
       "153             29.0           3.0             1801.0             1801.0   \n",
       "74              29.0           3.0             1801.0             1801.0   \n",
       "73              29.0           3.0             1801.0             1801.0   \n",
       "129             29.0           3.0             1801.0             1801.0   \n",
       "71              31.0           3.0             2217.0             2217.0   \n",
       "234             31.0           3.0             2217.0             2217.0   \n",
       "166             31.0           3.0             2322.0             2322.0   \n",
       "165             31.0           3.0             2322.0             2322.0   \n",
       "164             31.0           3.0             2217.0             2217.0   \n",
       "175             31.0           3.0             2217.0             2217.0   \n",
       "176             31.0           3.0             2322.0             2322.0   \n",
       "57              31.0           3.0             2217.0             2217.0   \n",
       "273             31.0           3.0             2217.0             2217.0   \n",
       "19              31.0           3.0             2217.0             2217.0   \n",
       "272             31.0           3.0             2217.0             2217.0   \n",
       "271             31.0           3.0             2217.0             2217.0   \n",
       "104             31.0           3.0             2217.0             2217.0   \n",
       "231             31.0           3.0             2217.0             2217.0   \n",
       "270             31.0           3.0             2217.0             2217.0   \n",
       "233             31.0           3.0             2217.0             2217.0   \n",
       "230             31.0           3.0             2217.0             2217.0   \n",
       "269             31.0           3.0             2217.0             2217.0   \n",
       "65              31.0           3.0             2217.0             2217.0   \n",
       "267             31.0           3.0             2217.0             2217.0   \n",
       "268             31.0           3.0             2217.0             2217.0   \n",
       "66              31.0           3.0             2217.0             2217.0   \n",
       "94              31.0           3.0             2217.0             2217.0   \n",
       "286            100.0           9.0            13324.0            13324.0   \n",
       "251             12.0           1.0             2528.0             2528.0   \n",
       "250             12.0           1.0             2528.0             2528.0   \n",
       "177            100.0           8.0            13201.0            13201.0   \n",
       "172            100.0           8.0            13201.0            13201.0   \n",
       "45             100.0           8.0            13201.0            13201.0   \n",
       "198            100.0           8.0            13201.0            13201.0   \n",
       "27             100.0           8.0            13201.0            13201.0   \n",
       "183            100.0           8.0            13201.0            13201.0   \n",
       "\n",
       "     avg_tactic_ground_avg  coverage  accuracy  \n",
       "130              62.103448      0.29  0.103448  \n",
       "26               62.103448      0.29  0.103448  \n",
       "236              62.103448      0.29  0.103448  \n",
       "12               62.103448      0.29  0.103448  \n",
       "108              62.103448      0.29  0.103448  \n",
       "75               62.103448      0.29  0.103448  \n",
       "117              62.103448      0.29  0.103448  \n",
       "135              62.103448      0.29  0.103448  \n",
       "134              62.103448      0.29  0.103448  \n",
       "133              62.103448      0.29  0.103448  \n",
       "33               62.103448      0.29  0.103448  \n",
       "131              62.103448      0.29  0.103448  \n",
       "128              62.103448      0.29  0.103448  \n",
       "76               62.103448      0.29  0.103448  \n",
       "153              62.103448      0.29  0.103448  \n",
       "74               62.103448      0.29  0.103448  \n",
       "73               62.103448      0.29  0.103448  \n",
       "129              62.103448      0.29  0.103448  \n",
       "71               71.516129      0.31  0.096774  \n",
       "234              71.516129      0.31  0.096774  \n",
       "166              74.903226      0.31  0.096774  \n",
       "165              74.903226      0.31  0.096774  \n",
       "164              71.516129      0.31  0.096774  \n",
       "175              71.516129      0.31  0.096774  \n",
       "176              74.903226      0.31  0.096774  \n",
       "57               71.516129      0.31  0.096774  \n",
       "273              71.516129      0.31  0.096774  \n",
       "19               71.516129      0.31  0.096774  \n",
       "272              71.516129      0.31  0.096774  \n",
       "271              71.516129      0.31  0.096774  \n",
       "104              71.516129      0.31  0.096774  \n",
       "231              71.516129      0.31  0.096774  \n",
       "270              71.516129      0.31  0.096774  \n",
       "233              71.516129      0.31  0.096774  \n",
       "230              71.516129      0.31  0.096774  \n",
       "269              71.516129      0.31  0.096774  \n",
       "65               71.516129      0.31  0.096774  \n",
       "267              71.516129      0.31  0.096774  \n",
       "268              71.516129      0.31  0.096774  \n",
       "66               71.516129      0.31  0.096774  \n",
       "94               71.516129      0.31  0.096774  \n",
       "286             133.240000      1.00  0.090000  \n",
       "251             210.666667      0.12  0.083333  \n",
       "250             210.666667      0.12  0.083333  \n",
       "177             132.010000      1.00  0.080000  \n",
       "172             132.010000      1.00  0.080000  \n",
       "45              132.010000      1.00  0.080000  \n",
       "198             132.010000      1.00  0.080000  \n",
       "27              132.010000      1.00  0.080000  \n",
       "183             132.010000      1.00  0.080000  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = new_m16.sort_values(by='accuracy', ascending=False)\n",
    "df[50:100]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyze Position Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['pos_move'] = df['position'] + \",\" + df['move']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df.groupby('pos_move')\n",
    "agg = df2.aggregate(np.sum)\n",
    "agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(agg['exec_time'], bins=100)\n",
    "plt.title('Histogram of Execution Time $(T_{1600},$ Stockfish 14, $P_{valid})$')\n",
    "plt.xlabel('Execution Time (s)')\n",
    "plt.ylabel('Frequency')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "high_exec = agg[agg['exec_time'] > 100]\n",
    "high_exec = high_exec.sort_values(by = ['exec_time'], ascending = [False])\n",
    "high_exec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "low_exec = agg[agg['exec_time'] <= 100]\n",
    "plt.hist(low_exec['exec_time'], bins=100)\n",
    "plt.title('Histogram of Execution Time $(T_{1600},$ Stockfish 14, $P_{valid})$')\n",
    "plt.xlabel('Execution Time (s)')\n",
    "plt.ylabel('Frequency')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_exs = [1, 5, 20, 100]\n",
    "time_vals = [27.5, 60 + 33, 60 * 6 + 33, 33 * 60 + 8]\n",
    "num_tactics = [9, 117, 469, 837]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(num_exs, time_vals)\n",
    "plt.xlabel('Number of train examples')\n",
    "plt.ylabel('Execution time (s)')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(num_exs, num_tactics)\n",
    "plt.xlabel('Number of train examples')\n",
    "plt.ylabel('Number of tactics learned')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(num_tactics, time_vals)\n",
    "plt.xlabel('Number of tactics learned')\n",
    "plt.ylabel('Execution time (s)')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate Dataset Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "total_games = 0\n",
    "valid_elo = 0\n",
    "total_pos = 0\n",
    "total_elo = 0\n",
    "game_count = Counter()\n",
    "\n",
    "pgn_path = 'tactics/data/lichess_db_standard_rated_2013-01.pgn'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "offsets = []\n",
    "handle = open(pgn_path)\n",
    "header = chess.pgn.read_headers(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "header.get('Termination')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import chess.pgn\n",
    "\n",
    "with open(pgn_path) as pgn:\n",
    "    while game := chess.pgn.read_game(pgn):\n",
    "        total_games += 1\n",
    "        white_elo = game.headers['WhiteElo']\n",
    "        black_elo = game.headers['BlackElo']\n",
    "        if '?' not in white_elo and '?' not in black_elo:\n",
    "            total_elo += int(white_elo)\n",
    "            total_elo += int(black_elo)\n",
    "            valid_elo += 1\n",
    "        game_count[game.headers['Event']] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Total games', total_games)\n",
    "avg_elo = total_elo / (2 * valid_elo)\n",
    "print('Average ELO', avg_elo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "variance = 0\n",
    "\n",
    "with open(pgn_path) as pgn:\n",
    "    while game := chess.pgn.read_game(pgn):\n",
    "        total_games += 1\n",
    "        white_elo = game.headers['WhiteElo']\n",
    "        black_elo = game.headers['BlackElo']\n",
    "        if '?' not in white_elo and '?' not in black_elo:\n",
    "            variance += (int(white_elo) - avg_elo) ** 2\n",
    "            variance += (int(black_elo) - avg_elo) ** 2\n",
    "        game_count[game.headers['Event']] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "print('SD', math.sqrt(variance / valid_elo))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import chess\n",
    "import chess.engine\n",
    "from tactics.util import *\n",
    "\n",
    "board = chess.Board('r1bqkb1r/ppp1pnpp/3p4/7Q/8/3B4/PPP2PPP/RNB2RK1 w kq - 0 10')\n",
    "move_names = [\n",
    "    'b1c3',\n",
    "    'c1g5',\n",
    "    'd3c4',\n",
    "    'f1e1'\n",
    "]\n",
    "moves = [chess.Move.from_uci(move_name) for move_name in move_names]\n",
    "engine_path = get_lc0_cmd(LC0, MAIA_1600) + ['--nncache=0']\n",
    "#engine_path = STOCKFISH\n",
    "mate_score = 2000\n",
    "n = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with chess.engine.SimpleEngine.popen_uci(engine_path) as engine:\n",
    "    tmp_analysis = engine.analyse(board, limit=chess.engine.Limit(nodes=1), multipv=n, game=object())\n",
    "    top_results = [(root['pv'][0], root['score'].relative.score(mate_score=mate_score)) for root in tmp_analysis]\n",
    "    top_n_results = top_results[:n]\n",
    "top_n_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with chess.engine.SimpleEngine.popen_uci(engine_path) as engine:\n",
    "    all_evals = get_evals(engine, board, board.legal_moves)\n",
    "    all_evals.sort(key=lambda eval: eval[1], reverse=True)\n",
    "    top_n_results = all_evals[:n]\n",
    "top_n_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_evals(engine: chess.engine.SimpleEngine, board: chess.Board, suggestions: List[chess.Move], mate_score: int=2000) -> List[Tuple[chess.Move, int]]:\n",
    "    \"Obtain engine evaluations for a list of moves in a given position\"\n",
    "\n",
    "    evals = []\n",
    "    for move in suggestions:\n",
    "        tmp_board = chess.Board(board.fen())\n",
    "        tmp_board.push(move)\n",
    "        if tmp_board.outcome() is not None:\n",
    "            move_score = mate_score if tmp_board.is_checkmate() else -mate_score\n",
    "            evals.append((move, move_score))\n",
    "            continue\n",
    "        limit = chess.engine.Limit(nodes=1)\n",
    "        # prev_eval = engine.analyse(board, limit=limit, game=object()) # https://stackoverflow.com/a/66251120\n",
    "        # print(prev_eval) \n",
    "        curr_eval = engine.analyse(tmp_board, limit=limit, game=object())\n",
    "        print(curr_eval)\n",
    "        orig_turn = board.turn\n",
    "        if 'pv' in curr_eval:\n",
    "            # prev_score = prev_eval['score'].pov(orig_turn)\n",
    "            curr_score = curr_eval['score'].pov(orig_turn)\n",
    "            move_score = curr_score.score(mate_score=mate_score)\n",
    "            evals.append((move, move_score))\n",
    "    return evals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with chess.engine.SimpleEngine.popen_uci(engine_path) as engine:\n",
    "    evals = get_evals(engine, board, moves)\n",
    "evals"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "5e324cb5850aaaec487f27f03ba9eba2477cd58aa23b6276a6301c881a1db4aa"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}