02_llm_comparison.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparing Models\n",
    "\n",
    "This notebook chooses the most appropriate prompt and prompt structure for the OCR correction. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jonno/clocrc/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "Downloading builder script: 100%|██████████| 4.49k/4.49k [00:00<00:00, 14.8MB/s]\n",
      "Downloading builder script: 100%|██████████| 5.60k/5.60k [00:00<00:00, 18.1MB/s]\n"
     ]
    }
   ],
   "source": [
    "#import config  # Import your config.py file this contains you openai api key\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from llm_comparison_toolkit import RateLimiter, get_response_openai, get_response_anthropic, get_response_replicate, create_config_dict_func, compare_request_configurations, generate_model_configs\n",
    "from evaluate import load\n",
    "from evaluation_funcs import evaluate_correction_performance, evaluate_correction_performance_folders, get_metric_error_reduction\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from helper_functions import files_to_df_func, files_to_df_core_func\n",
    "\n",
    "import re\n",
    "\n",
    "\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "load_dotenv()\n",
    "save_appendix = os.getenv(\"save_appendix\")\n",
    "save_figs = os.getenv(\"save_figs\")\n",
    "\n",
    "dev_data_folder = 'data/dev_data'\n",
    "dev_transcripts = os.path.join(dev_data_folder, 'dev_data_transcript')\n",
    "dev_raw_ocr_folder =  os.path.join(dev_data_folder,'dev_raw_ocr' )\n",
    "dev_system_message_folder = os.path.join(dev_data_folder,'dev_system_message_variants' )\n",
    "\n",
    "\n",
    "#NCSE\n",
    "\n",
    "ncse_folder = 'data/transcription_returned_ocr'\n",
    "ncse_articles_raw = os.path.join(ncse_folder, 'transcription_raw_ocr')\n",
    "ncse_articles_transcribed = os.path.join(ncse_folder, 'transcription_files') \n",
    "ncse_articles_results = os.path.join(ncse_folder, 'corrected_folder')\n",
    "\n",
    "#Overproof\n",
    "overproof_folder = 'data/overproof'\n",
    "\n",
    "smh_folder =  os.path.join(overproof_folder, 'SMH')\n",
    "smh_articles_raw = os.path.join(smh_folder, 'article_level', 'raw')\n",
    "smh_articles_transcribed = os.path.join(smh_folder, 'article_level', 'corrected') #in the dataset they are reffered to as corrected but this clashes with my naming convention\n",
    "smh_articles_results = os.path.join(smh_folder, 'results')\n",
    "\n",
    "ca_folder =  os.path.join(overproof_folder, 'CA')\n",
    "ca_articles_raw = os.path.join(ca_folder, 'article_level', 'raw')\n",
    "ca_articles_transcribed = os.path.join(ca_folder, 'article_level', 'corrected') #in the dataset they are reffered to as corrected but this clashes with my naming convention\n",
    "ca_articles_results = os.path.join(ca_folder, 'results')\n",
    "\n",
    "#load the dev and test sets for prompt development and selection\n",
    "dev_data_df = pd.read_csv(os.path.join(dev_data_folder,'dev_data_raw.csv'))\n",
    "\n",
    "\n",
    "#for saving data to be used in the analysis\n",
    "if not os.path.exists('data/analysis'):\n",
    "    os.makedirs('data/analysis')\n",
    "\n",
    "\n",
    "wer = load(\"wer\")\n",
    "cer = load(\"cer\")\n",
    "\n",
    "\n",
    "model_name_code = pd.Series(\n",
    "    {'Llama 2 70B':'llama2-70b-4096',\n",
    " 'Gemma 7B':'gemma-7b-it',\n",
    " 'Claude 3 Opus':'claude-3-opus-20240229',\n",
    " 'Claude 3 Haiku':'claude-3-haiku-20240307',\n",
    " 'GPT-4':'gpt-4-turbo-preview',\n",
    " 'GPT-3.5':'gpt-3.5-turbo',\n",
    " 'Mixtral 8x7B':'mixtral-8x7b-32768',\n",
    " \"Llama 3\":'meta-llama-3-70b-instruct',\n",
    " \"Llama 3 base\":'meta/meta-llama-3-70b',\n",
    " 'Overproof':'overproof'})\n",
    "\n",
    "\n",
    "eval_metric = 'CER'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate system prompt tests\n",
    "\n",
    "We evaluate the system prompts below to see if thre is any significant difference between the prompts\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluating models on the test set\n",
    "\n",
    "Having identified two different prompts and that the prompts appear to work better when places after the text we can now compare the different models\n",
    "\n",
    "\n",
    "The below code creates the basic configuration dictionaries for each model and then fills in the with the two different prompt messages creating a single list of all basic prompt/model configurations. It then calls all the LLM's and saves the results.\n",
    "This works in series so takes a while."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create API configurations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create the prompt/system message using the best performing from the previous section\n",
    "\n",
    "full_prompt = \"{content_html}\"+f\"\"\" \\n \\n \"\"\" + f\"\"\"You are an expert in post-OCR correction of documents. Using the context available from the text please recover the most likely original text from the corrupted OCR. The text is from an english newspaper in the 1800's. The text may be an advert or article and may be missing the beggining or end. Do not add any text, commentary, or lead in sentences beyond the recovered text. Do not add a title, or any introductions.\"\"\"\n",
    "\n",
    "instruct_prompt = \"{content_html}\"+f\"\"\" \\n \\n \"\"\" + f\"\"\"You are an expert in post-OCR correction of documents. Using the context available from the text please recover the most likely original text from the corrupted OCR. Do not add any text, commentary, or lead in sentences beyond the recovered text. Do not add a title, or any introductions.\"\"\"\n",
    "\n",
    "boros_basic  = \"{content_html}\"+f\"\"\" \\n \\n \"\"\" +\"Correct the text\"\n",
    "\n",
    "boros_complex  =\"{content_html}\"+f\"\"\" \\n \\n \"\"\" + f\"\"\"Please assist with reviewing and correcting errors in texts produced by automatic transcription (OCR) of historical documents.\n",
    "Your task is to carefully examine the following text and correct any mistakes introduced by the OCR software. \n",
    "Do not write anything else than the corrected text.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "groq_alt_endpoint = {'alt_endpoint':{'base_url':'https://api.groq.com/openai/v1',\n",
    "                     'api_key':os.getenv(\"GROQ_API_KEY\")}}\n",
    "\n",
    "basic_model_configs = pd.DataFrame({\n",
    "    'get_response_func': [get_response_openai, get_response_openai, get_response_openai, get_response_openai,\n",
    "                           get_response_anthropic, get_response_anthropic, \n",
    "                          get_response_openai,# get_response_openai, \n",
    "                          get_response_openai, get_response_replicate,# get_response_replicate, \n",
    "                          get_response_replicate], \n",
    "    'engine': ['gpt-3.5-turbo', 'gpt-4-turbo-preview', 'gpt-4o', 'gpt-4o-mini',\n",
    "               \"claude-3-haiku-20240307\", \"claude-3-opus-20240229\", \n",
    "               'mixtral-8x7b-32768',#'llama2-70b-4096', \n",
    "               'gemma-7b-it', 'meta/meta-llama-3-70b-instruct', 'meta/meta-llama-3-70b',\n",
    "                # 'meta/llama-2-70b'\n",
    "                 ],\n",
    "    'rate_limit':[160e3, 80e3, 160e3, 160e3,\n",
    "                  100e3, 40e3, 9e3, #15e3,\n",
    "                   15e3, 100e3,# 100e3,\n",
    "                     100e3],\n",
    "    'additional_args': [\n",
    "        {}, {}, {}, {}, \n",
    "        {},{},\n",
    "        groq_alt_endpoint, \n",
    "        #groq_alt_endpoint, \n",
    "        groq_alt_endpoint,\n",
    "        {}, {}#,{}\n",
    "    ]\n",
    "})\n",
    "\n",
    "full_model_configs = generate_model_configs(basic_model_configs, full_prompt, 'full')\n",
    "instruct_model_configs = generate_model_configs(basic_model_configs, instruct_prompt, 'instruct')\n",
    "\n",
    "#I think on reflection I only need to compare boros complex on gpt-4 as this was the best performer in their paper\n",
    "boros_configs_ncse = [\n",
    "    (get_response_openai, 'gpt-4-turbo-preview', boros_complex, \"boros_complex_\"),\n",
    "]\n",
    "\n",
    "boros_list = [\n",
    "    create_config_dict_func(\n",
    "        get_response_func=config[0],\n",
    "        rate_limiter=RateLimiter(80e3),\n",
    "        engine=config[1],\n",
    "        system_message_template=\"\",\n",
    "        prompt_template=config[2],\n",
    "        additional_args={\"response_name\": config[3]}\n",
    "    )\n",
    "    for config in boros_configs_ncse\n",
    "]\n",
    "\n",
    "model_configs = full_model_configs + instruct_model_configs + boros_list\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perform all API calls\n",
    "\n",
    "The below section is what actually calls the API, the code points to the folders where the raw OCR is and provides a path to where the corrected text should be stored."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NCSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "corrected_folder = ncse_articles_results\n",
    "#\n",
    "# This naming business needs to be cleaned up so the actual article ID is used. until then just have the following mess\n",
    "#\n",
    "# \n",
    "\n",
    "test_data_new = pd.read_csv(os.path.join(dev_data_folder,'transcription_raw_ocr.csv'))\n",
    "test_data_new = test_data_new.loc[test_data_new ['file_name'].isin(files_to_df_func(ncse_articles_transcribed )['file_name'])] #subset to just the data I have transcribed\n",
    "\n",
    "#This goes through the list of articles that have been transcribed, checks to see if there is a corrected version and if not generates it\n",
    "compare_request_configurations(test_data_new, model_configs, folder_path=corrected_folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Boros etal re-evaluation\n",
    "\n",
    "The post-OCR correction worked so well that the Boros etal prompt is being re-evaluated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "boros_complex  =\"{content}\"+f\"\"\" \\n \\n \"\"\" + f\"\"\"Please assist with reviewing and correcting errors in texts produced by automatic transcription (OCR) of historical documents.\n",
    "Your task is to carefully examine the following text and correct any mistakes introduced by the OCR software. \n",
    "Do not write anything else than the corrected text.\"\"\"\n",
    "\n",
    "\n",
    "boros_config = generate_model_configs(basic_model_configs.iloc[0:2, :], boros_complex, 'boros_complex')[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sydney Morning Herald\n",
    "\n",
    "This section performs the correction test on dataset 2 of the Overproof collection. This is data from the Sydney Morning Herald. In addition it re-tests the Boros et al."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "smh_data = files_to_df_core_func(smh_articles_raw )\n",
    "\n",
    "smh_data['content'] = smh_data['content'].str.replace('\\n', ' ')\n",
    "\n",
    "smh_data['id'] = smh_data['file_name'] # this is needed as the processing log uses an id to keep track of what has been processed and what hasn't to allow for easy restarts\n",
    "\n",
    "full_prompt_smh = \"{content}\"+f\"\"\" \\n \\n \"\"\" + f\"\"\"You are an expert in post-OCR correction of documents. Using the context available from the text please recover the most likely original text from the corrupted OCR. The text is from The Sydney Morning Herald 1842 -1950. The text may be an advert or article and may be missing the beggining or end. Do not add any text, commentary, or lead in sentences beyond the recovered text. Do not add a title, or any introductions.\"\"\"\n",
    "\n",
    "instruct_prompt_smh = \"{content}\"+f\"\"\" \\n \\n \"\"\" + f\"\"\"You are an expert in post-OCR correction of documents. Using the context available from the text please recover the most likely original text from the corrupted OCR. Do not add any text, commentary, or lead in sentences beyond the recovered text. Do not add a title, or any introductions.\"\"\"\n",
    "\n",
    "full_model_configs_smh = generate_model_configs(basic_model_configs, full_prompt_smh, 'full')\n",
    "instruct_model_configs_smh = generate_model_configs(basic_model_configs, instruct_prompt_smh, 'instruct')\n",
    "\n",
    "#Boros et al prompt added in as the overall system works so well, it seems strange theirs didn't work, this is a quick check\n",
    "smh_configs = full_model_configs_smh + instruct_model_configs_smh  + [boros_config ]\n",
    "\n",
    "corrected_folder_smh = smh_articles_results\n",
    "\n",
    "compare_request_configurations(smh_data, smh_configs, folder_path=corrected_folder_smh)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chronicalling America\n",
    "\n",
    "This section performs the correction test on dataset 2 of the Overproof collection. This is data from the Chronicalling America Dataset. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'basic_model_configs' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[3], line 12\u001b[0m\n\u001b[1;32m      7\u001b[0m full_prompt_ca \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{content}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m+\u001b[39m\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\u001b[38;5;124mYou are an expert in post-OCR correction of documents. Using the context available from the text please recover the most likely original text from the corrupted OCR. The text is from American Newspapers 1870 -1922. The text may be an advert or article and may be missing the beggining or end. Do not add any text, commentary, or lead in sentences beyond the recovered text. Do not add a title, or any introductions.\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m      9\u001b[0m instruct_prompt_ca \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{content}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m+\u001b[39m\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\u001b[38;5;124mYou are an expert in post-OCR correction of documents. Using the context available from the text please recover the most likely original text from the corrupted OCR. Do not add any text, commentary, or lead in sentences beyond the recovered text. Do not add a title, or any introductions.\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[0;32m---> 12\u001b[0m full_model_configs_ca \u001b[38;5;241m=\u001b[39m generate_model_configs(\u001b[43mbasic_model_configs\u001b[49m, full_prompt_ca, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfull\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     13\u001b[0m instruct_model_configs_ca \u001b[38;5;241m=\u001b[39m generate_model_configs(basic_model_configs, instruct_prompt_ca, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minstruct\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     15\u001b[0m ca_configs \u001b[38;5;241m=\u001b[39m full_model_configs_ca \u001b[38;5;241m+\u001b[39m instruct_model_configs_ca  \u001b[38;5;241m+\u001b[39m [boros_config ]\n",
      "\u001b[0;31mNameError\u001b[0m: name 'basic_model_configs' is not defined"
     ]
    }
   ],
   "source": [
    "ca_data = files_to_df_core_func(ca_articles_raw )\n",
    "\n",
    "ca_data['content'] = ca_data['content'].str.replace('\\n', ' ')\n",
    "\n",
    "ca_data['id'] = ca_data['file_name'] # this is needed as the processing log uses an id to keep track of what has been processed and what hasn't to allow for easy restarts\n",
    "\n",
    "full_prompt_ca = \"{content}\"+f\"\"\" \\n \\n \"\"\" + f\"\"\"You are an expert in post-OCR correction of documents. Using the context available from the text please recover the most likely original text from the corrupted OCR. The text is from American Newspapers 1870 -1922. The text may be an advert or article and may be missing the beggining or end. Do not add any text, commentary, or lead in sentences beyond the recovered text. Do not add a title, or any introductions.\"\"\"\n",
    "\n",
    "instruct_prompt_ca = \"{content}\"+f\"\"\" \\n \\n \"\"\" + f\"\"\"You are an expert in post-OCR correction of documents. Using the context available from the text please recover the most likely original text from the corrupted OCR. Do not add any text, commentary, or lead in sentences beyond the recovered text. Do not add a title, or any introductions.\"\"\"\n",
    "\n",
    "\n",
    "full_model_configs_ca = generate_model_configs(basic_model_configs, full_prompt_ca, 'full')\n",
    "instruct_model_configs_ca = generate_model_configs(basic_model_configs, instruct_prompt_ca, 'instruct')\n",
    "\n",
    "ca_configs = full_model_configs_ca + instruct_model_configs_ca  + [boros_config ]\n",
    "\n",
    "corrected_folder_ca = ca_articles_results\n",
    "\n",
    "compare_request_configurations(ca_data, ca_configs, folder_path=corrected_folder_ca)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate the prompts across all models\n",
    "\n",
    "On the smaller models, Full is worse than instruct on the larger models the reverse. Maybe this is related to ability to 'focus' or hold isntructions in memory?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>WER</th>\n",
       "      <th>CER</th>\n",
       "      <th>lev_dist</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>50%</th>\n",
       "      <th>50%</th>\n",
       "      <th>50%</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>instruct__meta-llama-3-70b</th>\n",
       "      <td>-102.41</td>\n",
       "      <td>-587.99</td>\n",
       "      <td>-592.88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__meta-llama-3-70b</th>\n",
       "      <td>-102.68</td>\n",
       "      <td>-481.02</td>\n",
       "      <td>-468.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__llama2-70b-4096</th>\n",
       "      <td>56.83</td>\n",
       "      <td>-28.27</td>\n",
       "      <td>-27.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__llama2-70b-4096</th>\n",
       "      <td>53.24</td>\n",
       "      <td>-11.20</td>\n",
       "      <td>-12.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gemma-7b-it</th>\n",
       "      <td>20.47</td>\n",
       "      <td>-2.34</td>\n",
       "      <td>-2.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gemma-7b-it</th>\n",
       "      <td>20.35</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__mixtral-8x7b-32768</th>\n",
       "      <td>48.72</td>\n",
       "      <td>6.60</td>\n",
       "      <td>5.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__mixtral-8x7b-32768</th>\n",
       "      <td>48.48</td>\n",
       "      <td>7.01</td>\n",
       "      <td>6.69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__claude-3-haiku-20240307</th>\n",
       "      <td>46.59</td>\n",
       "      <td>14.04</td>\n",
       "      <td>13.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__meta-llama-3-70b-instruct</th>\n",
       "      <td>46.43</td>\n",
       "      <td>16.15</td>\n",
       "      <td>14.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__meta-llama-3-70b-instruct</th>\n",
       "      <td>48.06</td>\n",
       "      <td>19.12</td>\n",
       "      <td>17.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__claude-3-haiku-20240307</th>\n",
       "      <td>55.64</td>\n",
       "      <td>27.80</td>\n",
       "      <td>27.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-3.5-turbo</th>\n",
       "      <td>62.41</td>\n",
       "      <td>37.65</td>\n",
       "      <td>36.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-3.5-turbo</th>\n",
       "      <td>67.98</td>\n",
       "      <td>39.38</td>\n",
       "      <td>38.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4o-mini</th>\n",
       "      <td>71.58</td>\n",
       "      <td>48.23</td>\n",
       "      <td>47.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4o-mini</th>\n",
       "      <td>71.58</td>\n",
       "      <td>51.28</td>\n",
       "      <td>50.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4-turbo-preview</th>\n",
       "      <td>77.27</td>\n",
       "      <td>59.82</td>\n",
       "      <td>57.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4-turbo-preview</th>\n",
       "      <td>77.78</td>\n",
       "      <td>60.42</td>\n",
       "      <td>56.37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boros_complex__gpt-4-turbo-preview</th>\n",
       "      <td>77.27</td>\n",
       "      <td>61.67</td>\n",
       "      <td>60.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4o</th>\n",
       "      <td>79.49</td>\n",
       "      <td>62.41</td>\n",
       "      <td>61.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__claude-3-opus-20240229</th>\n",
       "      <td>78.39</td>\n",
       "      <td>62.71</td>\n",
       "      <td>61.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__claude-3-opus-20240229</th>\n",
       "      <td>80.22</td>\n",
       "      <td>64.09</td>\n",
       "      <td>62.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4o</th>\n",
       "      <td>78.57</td>\n",
       "      <td>64.24</td>\n",
       "      <td>62.12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        WER     CER lev_dist\n",
       "                                        50%     50%      50%\n",
       "type                                                        \n",
       "instruct__meta-llama-3-70b          -102.41 -587.99  -592.88\n",
       "full__meta-llama-3-70b              -102.68 -481.02  -468.17\n",
       "full__llama2-70b-4096                 56.83  -28.27   -27.33\n",
       "instruct__llama2-70b-4096             53.24  -11.20   -12.44\n",
       "full__gemma-7b-it                     20.47   -2.34    -2.55\n",
       "instruct__gemma-7b-it                 20.35    0.15     0.12\n",
       "full__mixtral-8x7b-32768              48.72    6.60     5.74\n",
       "instruct__mixtral-8x7b-32768          48.48    7.01     6.69\n",
       "full__claude-3-haiku-20240307         46.59   14.04    13.70\n",
       "instruct__meta-llama-3-70b-instruct   46.43   16.15    14.76\n",
       "full__meta-llama-3-70b-instruct       48.06   19.12    17.32\n",
       "instruct__claude-3-haiku-20240307     55.64   27.80    27.46\n",
       "full__gpt-3.5-turbo                   62.41   37.65    36.28\n",
       "instruct__gpt-3.5-turbo               67.98   39.38    38.48\n",
       "full__gpt-4o-mini                     71.58   48.23    47.81\n",
       "instruct__gpt-4o-mini                 71.58   51.28    50.84\n",
       "instruct__gpt-4-turbo-preview         77.27   59.82    57.58\n",
       "full__gpt-4-turbo-preview             77.78   60.42    56.37\n",
       "boros_complex__gpt-4-turbo-preview    77.27   61.67    60.02\n",
       "instruct__gpt-4o                      79.49   62.41    61.75\n",
       "instruct__claude-3-opus-20240229      78.39   62.71    61.62\n",
       "full__claude-3-opus-20240229          80.22   64.09    62.78\n",
       "full__gpt-4o                          78.57   64.24    62.12"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##\n",
    "## double check get_metric_error_reduction\n",
    "##\n",
    "\n",
    "corrected_folder = ncse_articles_results \n",
    "\n",
    "gt_folder = ncse_articles_transcribed \n",
    "\n",
    "raw_ocr = ncse_articles_raw\n",
    "\n",
    "ncse_performance_eval =  evaluate_correction_performance_folders(corrected_folder, gt_folder, wer, cer)\n",
    "\n",
    "ncse_raw_ocr_eval =  evaluate_correction_performance(raw_ocr, gt_folder , wer, cer, 'raw_ocr')\n",
    "\n",
    "\n",
    "ncse_error_reduction = get_metric_error_reduction(ncse_performance_eval, ncse_raw_ocr_eval )\n",
    "\n",
    "ncse_error_reduction.groupby('type').describe().filter(regex = '50|median').round(2).sort_values((eval_metric, '50%'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          WER   CER lev_dist\n",
      "          50%   50%      50%\n",
      "type                        \n",
      "raw_ocr  0.63  0.17    341.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>WER</th>\n",
       "      <th>CER</th>\n",
       "      <th>lev_dist</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>50%</th>\n",
       "      <th>50%</th>\n",
       "      <th>50%</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>instruct__claude-3-opus-20240229</th>\n",
       "      <td>0.15</td>\n",
       "      <td>0.06</td>\n",
       "      <td>115.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__claude-3-opus-20240229</th>\n",
       "      <td>0.15</td>\n",
       "      <td>0.07</td>\n",
       "      <td>113.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4o</th>\n",
       "      <td>0.18</td>\n",
       "      <td>0.09</td>\n",
       "      <td>115.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boros_complex__gpt-4-turbo-preview</th>\n",
       "      <td>0.17</td>\n",
       "      <td>0.09</td>\n",
       "      <td>127.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4-turbo-preview</th>\n",
       "      <td>0.17</td>\n",
       "      <td>0.09</td>\n",
       "      <td>114.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4o</th>\n",
       "      <td>0.15</td>\n",
       "      <td>0.09</td>\n",
       "      <td>115.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4-turbo-preview</th>\n",
       "      <td>0.17</td>\n",
       "      <td>0.10</td>\n",
       "      <td>126.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4o-mini</th>\n",
       "      <td>0.23</td>\n",
       "      <td>0.12</td>\n",
       "      <td>153.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-3.5-turbo</th>\n",
       "      <td>0.28</td>\n",
       "      <td>0.13</td>\n",
       "      <td>172.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4o-mini</th>\n",
       "      <td>0.21</td>\n",
       "      <td>0.13</td>\n",
       "      <td>166.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__llama2-70b-4096</th>\n",
       "      <td>0.21</td>\n",
       "      <td>0.14</td>\n",
       "      <td>317.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__llama2-70b-4096</th>\n",
       "      <td>0.22</td>\n",
       "      <td>0.15</td>\n",
       "      <td>259.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-3.5-turbo</th>\n",
       "      <td>0.36</td>\n",
       "      <td>0.23</td>\n",
       "      <td>197.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__mixtral-8x7b-32768</th>\n",
       "      <td>0.46</td>\n",
       "      <td>0.32</td>\n",
       "      <td>337.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__claude-3-haiku-20240307</th>\n",
       "      <td>0.46</td>\n",
       "      <td>0.36</td>\n",
       "      <td>272.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__mixtral-8x7b-32768</th>\n",
       "      <td>0.51</td>\n",
       "      <td>0.36</td>\n",
       "      <td>394.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__meta-llama-3-70b-instruct</th>\n",
       "      <td>0.52</td>\n",
       "      <td>0.39</td>\n",
       "      <td>577.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__meta-llama-3-70b-instruct</th>\n",
       "      <td>0.52</td>\n",
       "      <td>0.40</td>\n",
       "      <td>492.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__claude-3-haiku-20240307</th>\n",
       "      <td>0.55</td>\n",
       "      <td>0.44</td>\n",
       "      <td>347.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gemma-7b-it</th>\n",
       "      <td>0.69</td>\n",
       "      <td>0.52</td>\n",
       "      <td>614.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gemma-7b-it</th>\n",
       "      <td>0.66</td>\n",
       "      <td>0.52</td>\n",
       "      <td>501.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__meta-llama-3-70b</th>\n",
       "      <td>0.98</td>\n",
       "      <td>0.91</td>\n",
       "      <td>1832.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__meta-llama-3-70b</th>\n",
       "      <td>0.99</td>\n",
       "      <td>0.94</td>\n",
       "      <td>1843.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      WER   CER lev_dist\n",
       "                                      50%   50%      50%\n",
       "type                                                    \n",
       "instruct__claude-3-opus-20240229     0.15  0.06    115.0\n",
       "full__claude-3-opus-20240229         0.15  0.07    113.0\n",
       "full__gpt-4o                         0.18  0.09    115.0\n",
       "boros_complex__gpt-4-turbo-preview   0.17  0.09    127.0\n",
       "full__gpt-4-turbo-preview            0.17  0.09    114.0\n",
       "instruct__gpt-4o                     0.15  0.09    115.0\n",
       "instruct__gpt-4-turbo-preview        0.17  0.10    126.0\n",
       "instruct__gpt-4o-mini                0.23  0.12    153.0\n",
       "instruct__gpt-3.5-turbo              0.28  0.13    172.0\n",
       "full__gpt-4o-mini                    0.21  0.13    166.0\n",
       "full__llama2-70b-4096                0.21  0.14    317.0\n",
       "instruct__llama2-70b-4096            0.22  0.15    259.5\n",
       "full__gpt-3.5-turbo                  0.36  0.23    197.0\n",
       "instruct__mixtral-8x7b-32768         0.46  0.32    337.0\n",
       "instruct__claude-3-haiku-20240307    0.46  0.36    272.0\n",
       "full__mixtral-8x7b-32768             0.51  0.36    394.0\n",
       "instruct__meta-llama-3-70b-instruct  0.52  0.39    577.0\n",
       "full__meta-llama-3-70b-instruct      0.52  0.40    492.0\n",
       "full__claude-3-haiku-20240307        0.55  0.44    347.0\n",
       "full__gemma-7b-it                    0.69  0.52    614.0\n",
       "instruct__gemma-7b-it                0.66  0.52    501.0\n",
       "full__meta-llama-3-70b               0.98  0.91   1832.0\n",
       "instruct__meta-llama-3-70b           0.99  0.94   1843.0"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(ncse_raw_ocr_eval.groupby('type').describe().filter(regex = '50|median').round(2).sort_values((eval_metric, '50%')))\n",
    "ncse_performance_eval.groupby('type').describe().filter(regex = '50|median').round(2).sort_values((eval_metric, '50%'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>WER</th>\n",
       "      <th>CER</th>\n",
       "      <th>lev_dist</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>50%</th>\n",
       "      <th>50%</th>\n",
       "      <th>50%</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>full__llama-2-70b</th>\n",
       "      <td>-89.25</td>\n",
       "      <td>-963.01</td>\n",
       "      <td>-914.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__llama-2-70b</th>\n",
       "      <td>-89.50</td>\n",
       "      <td>-958.60</td>\n",
       "      <td>-895.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__meta-llama-3-70b</th>\n",
       "      <td>-88.52</td>\n",
       "      <td>-867.47</td>\n",
       "      <td>-865.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__meta-llama-3-70b</th>\n",
       "      <td>-90.56</td>\n",
       "      <td>-864.63</td>\n",
       "      <td>-853.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gemma-7b-it</th>\n",
       "      <td>3.26</td>\n",
       "      <td>-35.65</td>\n",
       "      <td>-35.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__mixtral-8x7b-32768</th>\n",
       "      <td>17.39</td>\n",
       "      <td>-19.11</td>\n",
       "      <td>-20.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__mixtral-8x7b-32768</th>\n",
       "      <td>18.00</td>\n",
       "      <td>-14.63</td>\n",
       "      <td>-15.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gemma-7b-it</th>\n",
       "      <td>9.66</td>\n",
       "      <td>-12.93</td>\n",
       "      <td>-13.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__llama2-70b-4096</th>\n",
       "      <td>16.67</td>\n",
       "      <td>5.77</td>\n",
       "      <td>1.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__llama2-70b-4096</th>\n",
       "      <td>16.00</td>\n",
       "      <td>6.45</td>\n",
       "      <td>3.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__meta-llama-3-70b-instruct</th>\n",
       "      <td>18.75</td>\n",
       "      <td>12.90</td>\n",
       "      <td>8.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__meta-llama-3-70b-instruct</th>\n",
       "      <td>20.56</td>\n",
       "      <td>17.95</td>\n",
       "      <td>13.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>overproof</th>\n",
       "      <td>19.23</td>\n",
       "      <td>28.38</td>\n",
       "      <td>27.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__claude-3-haiku-20240307</th>\n",
       "      <td>26.92</td>\n",
       "      <td>35.71</td>\n",
       "      <td>33.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__claude-3-haiku-20240307</th>\n",
       "      <td>27.34</td>\n",
       "      <td>38.38</td>\n",
       "      <td>37.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-3.5-turbo</th>\n",
       "      <td>26.72</td>\n",
       "      <td>39.18</td>\n",
       "      <td>38.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4o-mini</th>\n",
       "      <td>28.44</td>\n",
       "      <td>41.03</td>\n",
       "      <td>40.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4-turbo-preview</th>\n",
       "      <td>30.20</td>\n",
       "      <td>41.79</td>\n",
       "      <td>41.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4-turbo-preview</th>\n",
       "      <td>30.00</td>\n",
       "      <td>42.08</td>\n",
       "      <td>41.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-3.5-turbo</th>\n",
       "      <td>27.75</td>\n",
       "      <td>42.86</td>\n",
       "      <td>41.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4o-mini</th>\n",
       "      <td>27.63</td>\n",
       "      <td>44.44</td>\n",
       "      <td>43.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__claude-3-opus-20240229</th>\n",
       "      <td>32.72</td>\n",
       "      <td>45.45</td>\n",
       "      <td>41.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boros_complex__gpt-4-turbo-preview</th>\n",
       "      <td>30.71</td>\n",
       "      <td>48.44</td>\n",
       "      <td>48.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4o</th>\n",
       "      <td>29.06</td>\n",
       "      <td>50.00</td>\n",
       "      <td>49.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__claude-3-opus-20240229</th>\n",
       "      <td>32.11</td>\n",
       "      <td>51.03</td>\n",
       "      <td>48.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4o</th>\n",
       "      <td>30.21</td>\n",
       "      <td>52.00</td>\n",
       "      <td>50.88</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       WER     CER lev_dist\n",
       "                                       50%     50%      50%\n",
       "type                                                       \n",
       "full__llama-2-70b                   -89.25 -963.01  -914.86\n",
       "instruct__llama-2-70b               -89.50 -958.60  -895.77\n",
       "instruct__meta-llama-3-70b          -88.52 -867.47  -865.27\n",
       "full__meta-llama-3-70b              -90.56 -864.63  -853.01\n",
       "full__gemma-7b-it                     3.26  -35.65   -35.34\n",
       "instruct__mixtral-8x7b-32768         17.39  -19.11   -20.00\n",
       "full__mixtral-8x7b-32768             18.00  -14.63   -15.24\n",
       "instruct__gemma-7b-it                 9.66  -12.93   -13.04\n",
       "full__llama2-70b-4096                16.67    5.77     1.85\n",
       "instruct__llama2-70b-4096            16.00    6.45     3.08\n",
       "full__meta-llama-3-70b-instruct      18.75   12.90     8.40\n",
       "instruct__meta-llama-3-70b-instruct  20.56   17.95    13.33\n",
       "overproof                            19.23   28.38    27.59\n",
       "instruct__claude-3-haiku-20240307    26.92   35.71    33.80\n",
       "full__claude-3-haiku-20240307        27.34   38.38    37.48\n",
       "full__gpt-3.5-turbo                  26.72   39.18    38.66\n",
       "full__gpt-4o-mini                    28.44   41.03    40.68\n",
       "instruct__gpt-4-turbo-preview        30.20   41.79    41.18\n",
       "full__gpt-4-turbo-preview            30.00   42.08    41.46\n",
       "instruct__gpt-3.5-turbo              27.75   42.86    41.46\n",
       "instruct__gpt-4o-mini                27.63   44.44    43.24\n",
       "instruct__claude-3-opus-20240229     32.72   45.45    41.57\n",
       "boros_complex__gpt-4-turbo-preview   30.71   48.44    48.23\n",
       "full__gpt-4o                         29.06   50.00    49.14\n",
       "full__claude-3-opus-20240229         32.11   51.03    48.78\n",
       "instruct__gpt-4o                     30.21   52.00    50.88"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrected_folder = smh_articles_results \n",
    "\n",
    "gt_folder = smh_articles_transcribed \n",
    "\n",
    "raw_ocr = smh_articles_raw\n",
    "\n",
    "smh_performance_eval =  evaluate_correction_performance_folders(corrected_folder, gt_folder, wer, cer)\n",
    "\n",
    "smh_raw_ocr_eval =  evaluate_correction_performance(raw_ocr, gt_folder , wer, cer, 'raw_ocr')\n",
    "\n",
    "smh_error_reduction = get_metric_error_reduction(smh_performance_eval, smh_raw_ocr_eval )\n",
    "\n",
    "smh_error_reduction.groupby('type').describe().filter(regex = '50|median').round(2).sort_values((eval_metric, '50%'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          WER   CER lev_dist\n",
      "          50%   50%      50%\n",
      "type                        \n",
      "raw_ocr  0.51  0.08    131.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>WER</th>\n",
       "      <th>CER</th>\n",
       "      <th>lev_dist</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>50%</th>\n",
       "      <th>50%</th>\n",
       "      <th>50%</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>boros_complex__gpt-4-turbo-preview</th>\n",
       "      <td>0.34</td>\n",
       "      <td>0.04</td>\n",
       "      <td>65.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__claude-3-haiku-20240307</th>\n",
       "      <td>0.37</td>\n",
       "      <td>0.04</td>\n",
       "      <td>73.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__claude-3-opus-20240229</th>\n",
       "      <td>0.34</td>\n",
       "      <td>0.04</td>\n",
       "      <td>69.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4o-mini</th>\n",
       "      <td>0.35</td>\n",
       "      <td>0.04</td>\n",
       "      <td>75.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-3.5-turbo</th>\n",
       "      <td>0.35</td>\n",
       "      <td>0.04</td>\n",
       "      <td>87.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4o</th>\n",
       "      <td>0.34</td>\n",
       "      <td>0.04</td>\n",
       "      <td>62.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4o</th>\n",
       "      <td>0.34</td>\n",
       "      <td>0.04</td>\n",
       "      <td>62.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4o-mini</th>\n",
       "      <td>0.35</td>\n",
       "      <td>0.04</td>\n",
       "      <td>76.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-3.5-turbo</th>\n",
       "      <td>0.35</td>\n",
       "      <td>0.04</td>\n",
       "      <td>78.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__claude-3-opus-20240229</th>\n",
       "      <td>0.34</td>\n",
       "      <td>0.04</td>\n",
       "      <td>71.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4-turbo-preview</th>\n",
       "      <td>0.34</td>\n",
       "      <td>0.05</td>\n",
       "      <td>75.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__claude-3-haiku-20240307</th>\n",
       "      <td>0.36</td>\n",
       "      <td>0.05</td>\n",
       "      <td>81.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>overproof</th>\n",
       "      <td>0.40</td>\n",
       "      <td>0.05</td>\n",
       "      <td>88.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4-turbo-preview</th>\n",
       "      <td>0.34</td>\n",
       "      <td>0.05</td>\n",
       "      <td>74.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__meta-llama-3-70b-instruct</th>\n",
       "      <td>0.42</td>\n",
       "      <td>0.07</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__llama2-70b-4096</th>\n",
       "      <td>0.40</td>\n",
       "      <td>0.07</td>\n",
       "      <td>115.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__llama2-70b-4096</th>\n",
       "      <td>0.40</td>\n",
       "      <td>0.07</td>\n",
       "      <td>117.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__meta-llama-3-70b-instruct</th>\n",
       "      <td>0.41</td>\n",
       "      <td>0.07</td>\n",
       "      <td>83.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__mixtral-8x7b-32768</th>\n",
       "      <td>0.40</td>\n",
       "      <td>0.10</td>\n",
       "      <td>159.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__mixtral-8x7b-32768</th>\n",
       "      <td>0.41</td>\n",
       "      <td>0.10</td>\n",
       "      <td>147.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gemma-7b-it</th>\n",
       "      <td>0.48</td>\n",
       "      <td>0.12</td>\n",
       "      <td>160.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gemma-7b-it</th>\n",
       "      <td>0.51</td>\n",
       "      <td>0.17</td>\n",
       "      <td>212.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__meta-llama-3-70b</th>\n",
       "      <td>0.97</td>\n",
       "      <td>0.84</td>\n",
       "      <td>1307.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__meta-llama-3-70b</th>\n",
       "      <td>0.97</td>\n",
       "      <td>0.87</td>\n",
       "      <td>1343.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__llama-2-70b</th>\n",
       "      <td>0.97</td>\n",
       "      <td>0.89</td>\n",
       "      <td>1339.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__llama-2-70b</th>\n",
       "      <td>0.98</td>\n",
       "      <td>0.89</td>\n",
       "      <td>1324.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      WER   CER lev_dist\n",
       "                                      50%   50%      50%\n",
       "type                                                    \n",
       "boros_complex__gpt-4-turbo-preview   0.34  0.04     65.0\n",
       "full__claude-3-haiku-20240307        0.37  0.04     73.0\n",
       "full__claude-3-opus-20240229         0.34  0.04     69.0\n",
       "instruct__gpt-4o-mini                0.35  0.04     75.0\n",
       "full__gpt-3.5-turbo                  0.35  0.04     87.0\n",
       "instruct__gpt-4o                     0.34  0.04     62.0\n",
       "full__gpt-4o                         0.34  0.04     62.0\n",
       "full__gpt-4o-mini                    0.35  0.04     76.0\n",
       "instruct__gpt-3.5-turbo              0.35  0.04     78.0\n",
       "instruct__claude-3-opus-20240229     0.34  0.04     71.0\n",
       "instruct__gpt-4-turbo-preview        0.34  0.05     75.0\n",
       "instruct__claude-3-haiku-20240307    0.36  0.05     81.0\n",
       "overproof                            0.40  0.05     88.0\n",
       "full__gpt-4-turbo-preview            0.34  0.05     74.0\n",
       "full__meta-llama-3-70b-instruct      0.42  0.07    100.0\n",
       "full__llama2-70b-4096                0.40  0.07    115.0\n",
       "instruct__llama2-70b-4096            0.40  0.07    117.0\n",
       "instruct__meta-llama-3-70b-instruct  0.41  0.07     83.0\n",
       "instruct__mixtral-8x7b-32768         0.40  0.10    159.0\n",
       "full__mixtral-8x7b-32768             0.41  0.10    147.0\n",
       "instruct__gemma-7b-it                0.48  0.12    160.0\n",
       "full__gemma-7b-it                    0.51  0.17    212.0\n",
       "full__meta-llama-3-70b               0.97  0.84   1307.0\n",
       "instruct__meta-llama-3-70b           0.97  0.87   1343.0\n",
       "full__llama-2-70b                    0.97  0.89   1339.0\n",
       "instruct__llama-2-70b                0.98  0.89   1324.0"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(smh_raw_ocr_eval.groupby('type').describe().filter(regex = '50|median').round(2).sort_values((eval_metric, '50%')))\n",
    "\n",
    "smh_performance_eval.groupby('type').describe().filter(regex = '50|median').round(2).sort_values((eval_metric, '50%'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>WER</th>\n",
       "      <th>CER</th>\n",
       "      <th>lev_dist</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>50%</th>\n",
       "      <th>50%</th>\n",
       "      <th>50%</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>full__llama-2-70b</th>\n",
       "      <td>-59.57</td>\n",
       "      <td>-784.90</td>\n",
       "      <td>-761.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__llama-2-70b</th>\n",
       "      <td>-61.06</td>\n",
       "      <td>-752.31</td>\n",
       "      <td>-741.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__meta-llama-3-70b</th>\n",
       "      <td>-71.75</td>\n",
       "      <td>-741.39</td>\n",
       "      <td>-739.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__meta-llama-3-70b</th>\n",
       "      <td>-63.14</td>\n",
       "      <td>-729.38</td>\n",
       "      <td>-725.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gemma-7b-it</th>\n",
       "      <td>5.66</td>\n",
       "      <td>-41.00</td>\n",
       "      <td>-42.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gemma-7b-it</th>\n",
       "      <td>3.04</td>\n",
       "      <td>-38.01</td>\n",
       "      <td>-38.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__mixtral-8x7b-32768</th>\n",
       "      <td>13.72</td>\n",
       "      <td>-22.07</td>\n",
       "      <td>-22.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__mixtral-8x7b-32768</th>\n",
       "      <td>15.72</td>\n",
       "      <td>-16.30</td>\n",
       "      <td>-14.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__meta-llama-3-70b-instruct</th>\n",
       "      <td>9.74</td>\n",
       "      <td>-9.42</td>\n",
       "      <td>-10.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__llama2-70b-4096</th>\n",
       "      <td>10.86</td>\n",
       "      <td>-6.48</td>\n",
       "      <td>-7.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__llama2-70b-4096</th>\n",
       "      <td>10.19</td>\n",
       "      <td>-4.90</td>\n",
       "      <td>-8.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__meta-llama-3-70b-instruct</th>\n",
       "      <td>12.91</td>\n",
       "      <td>14.24</td>\n",
       "      <td>10.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__claude-3-haiku-20240307</th>\n",
       "      <td>14.97</td>\n",
       "      <td>26.01</td>\n",
       "      <td>25.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>overproof</th>\n",
       "      <td>21.63</td>\n",
       "      <td>34.59</td>\n",
       "      <td>34.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__claude-3-haiku-20240307</th>\n",
       "      <td>17.41</td>\n",
       "      <td>34.73</td>\n",
       "      <td>34.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4-turbo-preview</th>\n",
       "      <td>20.98</td>\n",
       "      <td>37.57</td>\n",
       "      <td>37.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4-turbo-preview</th>\n",
       "      <td>21.84</td>\n",
       "      <td>38.18</td>\n",
       "      <td>37.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4o-mini</th>\n",
       "      <td>23.40</td>\n",
       "      <td>39.09</td>\n",
       "      <td>37.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4o-mini</th>\n",
       "      <td>23.56</td>\n",
       "      <td>40.25</td>\n",
       "      <td>40.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-3.5-turbo</th>\n",
       "      <td>24.62</td>\n",
       "      <td>44.05</td>\n",
       "      <td>44.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-3.5-turbo</th>\n",
       "      <td>22.90</td>\n",
       "      <td>44.22</td>\n",
       "      <td>43.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4o</th>\n",
       "      <td>22.42</td>\n",
       "      <td>45.52</td>\n",
       "      <td>44.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boros_complex__gpt-4-turbo-preview</th>\n",
       "      <td>24.02</td>\n",
       "      <td>45.55</td>\n",
       "      <td>45.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4o</th>\n",
       "      <td>24.05</td>\n",
       "      <td>45.68</td>\n",
       "      <td>44.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__claude-3-opus-20240229</th>\n",
       "      <td>24.26</td>\n",
       "      <td>47.01</td>\n",
       "      <td>41.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__claude-3-opus-20240229</th>\n",
       "      <td>24.61</td>\n",
       "      <td>48.23</td>\n",
       "      <td>43.44</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       WER     CER lev_dist\n",
       "                                       50%     50%      50%\n",
       "type                                                       \n",
       "full__llama-2-70b                   -59.57 -784.90  -761.72\n",
       "instruct__llama-2-70b               -61.06 -752.31  -741.10\n",
       "full__meta-llama-3-70b              -71.75 -741.39  -739.31\n",
       "instruct__meta-llama-3-70b          -63.14 -729.38  -725.21\n",
       "instruct__gemma-7b-it                 5.66  -41.00   -42.38\n",
       "full__gemma-7b-it                     3.04  -38.01   -38.15\n",
       "instruct__mixtral-8x7b-32768         13.72  -22.07   -22.36\n",
       "full__mixtral-8x7b-32768             15.72  -16.30   -14.86\n",
       "full__meta-llama-3-70b-instruct       9.74   -9.42   -10.51\n",
       "instruct__llama2-70b-4096            10.86   -6.48    -7.94\n",
       "full__llama2-70b-4096                10.19   -4.90    -8.14\n",
       "instruct__meta-llama-3-70b-instruct  12.91   14.24    10.90\n",
       "full__claude-3-haiku-20240307        14.97   26.01    25.27\n",
       "overproof                            21.63   34.59    34.59\n",
       "instruct__claude-3-haiku-20240307    17.41   34.73    34.73\n",
       "instruct__gpt-4-turbo-preview        20.98   37.57    37.45\n",
       "full__gpt-4-turbo-preview            21.84   38.18    37.70\n",
       "full__gpt-4o-mini                    23.40   39.09    37.95\n",
       "instruct__gpt-4o-mini                23.56   40.25    40.13\n",
       "instruct__gpt-3.5-turbo              24.62   44.05    44.05\n",
       "full__gpt-3.5-turbo                  22.90   44.22    43.99\n",
       "full__gpt-4o                         22.42   45.52    44.06\n",
       "boros_complex__gpt-4-turbo-preview   24.02   45.55    45.19\n",
       "instruct__gpt-4o                     24.05   45.68    44.74\n",
       "instruct__claude-3-opus-20240229     24.26   47.01    41.18\n",
       "full__claude-3-opus-20240229         24.61   48.23    43.44"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrected_folder = ca_articles_results \n",
    "\n",
    "gt_folder = ca_articles_transcribed \n",
    "\n",
    "raw_ocr = ca_articles_raw\n",
    "\n",
    "ca_performance_eval =  evaluate_correction_performance_folders(corrected_folder, gt_folder, wer, cer)\n",
    "\n",
    "ca_raw_ocr_eval =  evaluate_correction_performance(raw_ocr, gt_folder , wer, cer, 'raw_ocr')\n",
    "\n",
    "ca_error_reduction = get_metric_error_reduction(ca_performance_eval, ca_raw_ocr_eval )\n",
    "\n",
    "ca_error_reduction.groupby('type').describe().filter(regex = '50|median').round(2).sort_values((eval_metric, '50%'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          WER  CER lev_dist\n",
      "          50%  50%      50%\n",
      "type                       \n",
      "raw_ocr  0.62  0.1    198.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>WER</th>\n",
       "      <th>CER</th>\n",
       "      <th>lev_dist</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>50%</th>\n",
       "      <th>50%</th>\n",
       "      <th>50%</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>boros_complex__gpt-4-turbo-preview</th>\n",
       "      <td>0.45</td>\n",
       "      <td>0.05</td>\n",
       "      <td>107.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__claude-3-opus-20240229</th>\n",
       "      <td>0.43</td>\n",
       "      <td>0.05</td>\n",
       "      <td>117.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4o</th>\n",
       "      <td>0.44</td>\n",
       "      <td>0.05</td>\n",
       "      <td>101.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4o</th>\n",
       "      <td>0.46</td>\n",
       "      <td>0.05</td>\n",
       "      <td>102.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-3.5-turbo</th>\n",
       "      <td>0.45</td>\n",
       "      <td>0.05</td>\n",
       "      <td>100.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__claude-3-opus-20240229</th>\n",
       "      <td>0.44</td>\n",
       "      <td>0.05</td>\n",
       "      <td>119.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4o-mini</th>\n",
       "      <td>0.46</td>\n",
       "      <td>0.06</td>\n",
       "      <td>114.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-3.5-turbo</th>\n",
       "      <td>0.46</td>\n",
       "      <td>0.06</td>\n",
       "      <td>108.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4-turbo-preview</th>\n",
       "      <td>0.45</td>\n",
       "      <td>0.06</td>\n",
       "      <td>120.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gpt-4o-mini</th>\n",
       "      <td>0.46</td>\n",
       "      <td>0.06</td>\n",
       "      <td>125.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gpt-4-turbo-preview</th>\n",
       "      <td>0.45</td>\n",
       "      <td>0.06</td>\n",
       "      <td>118.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__claude-3-haiku-20240307</th>\n",
       "      <td>0.47</td>\n",
       "      <td>0.06</td>\n",
       "      <td>112.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>overproof</th>\n",
       "      <td>0.47</td>\n",
       "      <td>0.07</td>\n",
       "      <td>111.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__claude-3-haiku-20240307</th>\n",
       "      <td>0.48</td>\n",
       "      <td>0.07</td>\n",
       "      <td>123.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__meta-llama-3-70b-instruct</th>\n",
       "      <td>0.54</td>\n",
       "      <td>0.10</td>\n",
       "      <td>132.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__meta-llama-3-70b-instruct</th>\n",
       "      <td>0.55</td>\n",
       "      <td>0.11</td>\n",
       "      <td>145.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__llama2-70b-4096</th>\n",
       "      <td>0.50</td>\n",
       "      <td>0.11</td>\n",
       "      <td>203.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__llama2-70b-4096</th>\n",
       "      <td>0.51</td>\n",
       "      <td>0.11</td>\n",
       "      <td>195.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__mixtral-8x7b-32768</th>\n",
       "      <td>0.50</td>\n",
       "      <td>0.12</td>\n",
       "      <td>234.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__mixtral-8x7b-32768</th>\n",
       "      <td>0.50</td>\n",
       "      <td>0.12</td>\n",
       "      <td>230.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__gemma-7b-it</th>\n",
       "      <td>0.57</td>\n",
       "      <td>0.15</td>\n",
       "      <td>239.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__gemma-7b-it</th>\n",
       "      <td>0.59</td>\n",
       "      <td>0.15</td>\n",
       "      <td>299.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__meta-llama-3-70b</th>\n",
       "      <td>0.98</td>\n",
       "      <td>0.86</td>\n",
       "      <td>1784.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__meta-llama-3-70b</th>\n",
       "      <td>0.98</td>\n",
       "      <td>0.89</td>\n",
       "      <td>1696.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instruct__llama-2-70b</th>\n",
       "      <td>0.98</td>\n",
       "      <td>0.91</td>\n",
       "      <td>1608.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>full__llama-2-70b</th>\n",
       "      <td>0.98</td>\n",
       "      <td>0.92</td>\n",
       "      <td>1589.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      WER   CER lev_dist\n",
       "                                      50%   50%      50%\n",
       "type                                                    \n",
       "boros_complex__gpt-4-turbo-preview   0.45  0.05    107.0\n",
       "full__claude-3-opus-20240229         0.43  0.05    117.0\n",
       "instruct__gpt-4o                     0.44  0.05    101.5\n",
       "full__gpt-4o                         0.46  0.05    102.5\n",
       "instruct__gpt-3.5-turbo              0.45  0.05    100.5\n",
       "instruct__claude-3-opus-20240229     0.44  0.05    119.0\n",
       "instruct__gpt-4o-mini                0.46  0.06    114.0\n",
       "full__gpt-3.5-turbo                  0.46  0.06    108.0\n",
       "full__gpt-4-turbo-preview            0.45  0.06    120.5\n",
       "full__gpt-4o-mini                    0.46  0.06    125.0\n",
       "instruct__gpt-4-turbo-preview        0.45  0.06    118.0\n",
       "instruct__claude-3-haiku-20240307    0.47  0.06    112.0\n",
       "overproof                            0.47  0.07    111.0\n",
       "full__claude-3-haiku-20240307        0.48  0.07    123.0\n",
       "instruct__meta-llama-3-70b-instruct  0.54  0.10    132.0\n",
       "full__meta-llama-3-70b-instruct      0.55  0.11    145.5\n",
       "full__llama2-70b-4096                0.50  0.11    203.0\n",
       "instruct__llama2-70b-4096            0.51  0.11    195.0\n",
       "instruct__mixtral-8x7b-32768         0.50  0.12    234.5\n",
       "full__mixtral-8x7b-32768             0.50  0.12    230.5\n",
       "instruct__gemma-7b-it                0.57  0.15    239.5\n",
       "full__gemma-7b-it                    0.59  0.15    299.0\n",
       "full__meta-llama-3-70b               0.98  0.86   1784.5\n",
       "instruct__meta-llama-3-70b           0.98  0.89   1696.0\n",
       "instruct__llama-2-70b                0.98  0.91   1608.5\n",
       "full__llama-2-70b                    0.98  0.92   1589.5"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(ca_raw_ocr_eval.groupby('type').describe().filter(regex = '50|median').round(2).sort_values((eval_metric, '50%')))\n",
    "\n",
    "ca_performance_eval.groupby('type').describe().filter(regex = '50|median').round(2).sort_values((eval_metric, '50%'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>prompt</th>\n",
       "      <th>NCSE</th>\n",
       "      <th>SMH</th>\n",
       "      <th>CA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Llama 2 70B</td>\n",
       "      <td>full</td>\n",
       "      <td>-28.3</td>\n",
       "      <td>5.8</td>\n",
       "      <td>-4.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Llama 2 70B</td>\n",
       "      <td>instruct</td>\n",
       "      <td>-11.2</td>\n",
       "      <td>6.5</td>\n",
       "      <td>-6.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Gemma 7B</td>\n",
       "      <td>full</td>\n",
       "      <td>-2.3</td>\n",
       "      <td>-35.7</td>\n",
       "      <td>-38.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Gemma 7B</td>\n",
       "      <td>instruct</td>\n",
       "      <td>0.1</td>\n",
       "      <td>-12.9</td>\n",
       "      <td>-41.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Mixtral 8x7B</td>\n",
       "      <td>full</td>\n",
       "      <td>6.6</td>\n",
       "      <td>-14.6</td>\n",
       "      <td>-16.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Mixtral 8x7B</td>\n",
       "      <td>instruct</td>\n",
       "      <td>7.0</td>\n",
       "      <td>-19.1</td>\n",
       "      <td>-22.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Claude 3 Haiku</td>\n",
       "      <td>full</td>\n",
       "      <td>14.0</td>\n",
       "      <td>38.4</td>\n",
       "      <td>26.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Llama 3</td>\n",
       "      <td>instruct</td>\n",
       "      <td>16.1</td>\n",
       "      <td>17.9</td>\n",
       "      <td>14.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Llama 3</td>\n",
       "      <td>full</td>\n",
       "      <td>19.1</td>\n",
       "      <td>12.9</td>\n",
       "      <td>-9.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Claude 3 Haiku</td>\n",
       "      <td>instruct</td>\n",
       "      <td>27.8</td>\n",
       "      <td>35.7</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>GPT-3.5</td>\n",
       "      <td>full</td>\n",
       "      <td>37.7</td>\n",
       "      <td>39.2</td>\n",
       "      <td>44.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>GPT-3.5</td>\n",
       "      <td>instruct</td>\n",
       "      <td>39.4</td>\n",
       "      <td>42.9</td>\n",
       "      <td>44.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>GPT-4</td>\n",
       "      <td>instruct</td>\n",
       "      <td>59.8</td>\n",
       "      <td>41.8</td>\n",
       "      <td>37.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>GPT-4</td>\n",
       "      <td>full</td>\n",
       "      <td>60.4</td>\n",
       "      <td>42.1</td>\n",
       "      <td>38.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>GPT-4 Boros</td>\n",
       "      <td>boros</td>\n",
       "      <td>61.7</td>\n",
       "      <td>48.4</td>\n",
       "      <td>45.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Claude 3 Opus</td>\n",
       "      <td>instruct</td>\n",
       "      <td>62.7</td>\n",
       "      <td>45.5</td>\n",
       "      <td>47.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Claude 3 Opus</td>\n",
       "      <td>full</td>\n",
       "      <td>64.1</td>\n",
       "      <td>51.0</td>\n",
       "      <td>48.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Overproof</td>\n",
       "      <td>overproof</td>\n",
       "      <td>NaN</td>\n",
       "      <td>28.4</td>\n",
       "      <td>34.6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Model     prompt  NCSE   SMH    CA\n",
       "0      Llama 2 70B       full -28.3   5.8  -4.9\n",
       "1      Llama 2 70B   instruct -11.2   6.5  -6.5\n",
       "2         Gemma 7B       full  -2.3 -35.7 -38.0\n",
       "3         Gemma 7B   instruct   0.1 -12.9 -41.0\n",
       "4     Mixtral 8x7B       full   6.6 -14.6 -16.3\n",
       "5     Mixtral 8x7B   instruct   7.0 -19.1 -22.1\n",
       "6   Claude 3 Haiku       full  14.0  38.4  26.0\n",
       "7          Llama 3   instruct  16.1  17.9  14.2\n",
       "8          Llama 3       full  19.1  12.9  -9.4\n",
       "9   Claude 3 Haiku   instruct  27.8  35.7  34.7\n",
       "10         GPT-3.5       full  37.7  39.2  44.2\n",
       "11         GPT-3.5   instruct  39.4  42.9  44.1\n",
       "12           GPT-4   instruct  59.8  41.8  37.6\n",
       "13           GPT-4       full  60.4  42.1  38.2\n",
       "14     GPT-4 Boros      boros  61.7  48.4  45.6\n",
       "15   Claude 3 Opus   instruct  62.7  45.5  47.0\n",
       "16   Claude 3 Opus       full  64.1  51.0  48.2\n",
       "17       Overproof  overproof   NaN  28.4  34.6"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ncse_median = ncse_error_reduction.groupby('type')[eval_metric].median().round(1)\n",
    "smh_median = smh_error_reduction.groupby('type')[eval_metric].median().round(1)\n",
    "ca_median = ca_error_reduction.groupby('type')[eval_metric].median().round(1)\n",
    "\n",
    "# Combine the results into a new dataframe\n",
    "result_df = pd.DataFrame({\n",
    "    'NCSE': ncse_median,\n",
    "    'SMH': smh_median,\n",
    "    'CA': ca_median\n",
    "})\n",
    "\n",
    "# Reset the index to make 'type' a regular column\n",
    "result_df = result_df.reset_index()\n",
    "\n",
    "result_df = result_df.loc[~result_df['type'].isin(['boros_basic__gpt-4-turbo-preview',  'claude_temp_claude-3-opus-20240229'])]\n",
    "\n",
    "result_df['model'] = result_df['type'].str.split('_').str[-1]\n",
    "result_df['prompt'] = result_df['type'].str.split('_').str[0]\n",
    "\n",
    "result_df = result_df.sort_values('NCSE').merge(model_name_code.reset_index().rename(columns={0: 'model'}), on='model').rename(columns={'index': 'Model'})\n",
    "\n",
    "result_df['Model'] = np.where(result_df['prompt']=='boros', result_df['Model'] + \" Boros\", result_df['Model'])\n",
    "\n",
    "#result_df = result_df.loc[~(result_df['Model'].str.contains('Llama 2'))]\n",
    "\n",
    "result_df[['Model', 'prompt', 'NCSE', 'SMH', 'CA']].sort_values(['NCSE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def render_latex_with_formatting(df, caption, label):\n",
    "\n",
    "    df = df.copy()\n",
    "    # Format the 'Total' row by appending \\textbf{} to each element\n",
    "    #df.iloc[-1] = df.iloc[-1].apply(lambda x: '\\\\textbf{' + str(x) + '}')\n",
    "    \n",
    "    # Convert DataFrame to LaTeX\n",
    "    latex_table = df.to_latex(\n",
    "        index=False,\n",
    "        float_format=\"%.2f\" ,\n",
    "        escape=False,  # Important to render LaTeX commands within the table properly\n",
    "        column_format='p{5cm}cccc',  # One left-aligned column followed by four centered columns\n",
    "        bold_rows=True,  # Bold the headers\n",
    "        caption=caption,\n",
    "        label=label\n",
    "    )\n",
    "    return latex_table\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{table}\n",
      "\\caption{Model performance across the datasets measured in Error Reduction Percentage, higher is better.There is significant variation in how well the LMs are able to perform post-OCR correction, and significant differences between prompts for certain models.}\n",
      "\\label{tab:results}\n",
      "\\begin{tabular}{p{5cm}cccc}\n",
      "\\toprule\n",
      "Model & NCSE & SMH & CA \\\\\n",
      "\\midrule\n",
      "Claude 3 Haiku & 14.00 & 38.40 & 26.00 \\\\\n",
      "Claude 3 Opus & 64.10 & 51.00 & 48.20 \\\\\n",
      "GPT-3.5 & 37.70 & 39.20 & 44.20 \\\\\n",
      "GPT-4 & 60.40 & 42.10 & 38.20 \\\\\n",
      "GPT-4 Boros & 61.70 & 48.40 & 45.60 \\\\\n",
      "Gemma 7B & -2.30 & -35.70 & -38.00 \\\\\n",
      "Llama 3 & 19.10 & 12.90 & -9.40 \\\\\n",
      "Mixtral 8x7B & 6.60 & -14.60 & -16.30 \\\\\n",
      "Overproof & NaN & 28.40 & 34.60 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\\end{table}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "results_tab = result_df.loc[(result_df['prompt']!='instruct') & ~(result_df['Model'].str.contains('Llama 2')), ['Model',  'NCSE', 'SMH', 'CA']].sort_values(['Model'])\n",
    "\n",
    "results_tab = render_latex_with_formatting(results_tab,   \n",
    "                            'Model performance across the datasets measured in Error Reduction Percentage, higher is better.There is significant variation in how well the LMs are able to perform post-OCR correction, and significant differences between prompts for certain models.',\n",
    "                            'tab:results'  )\n",
    "\n",
    "print(results_tab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Two metric table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "def create_combined_metric_table(error_reduction_df, performance_eval_df, model_name_code, agg_method='median'):\n",
    "    \"\"\"\n",
    "    Creates a combined DataFrame with error reduction and performance evaluation metrics.\n",
    "    \n",
    "    Parameters:\n",
    "    - error_reduction_df: DataFrame with error reduction data for all datasets\n",
    "    - performance_eval_df: DataFrame with performance evaluation data for all datasets\n",
    "    - model_name_code: DataFrame with model names and codes for merging\n",
    "    - agg_method: str, either 'median' or 'mean' for aggregation method\n",
    "    \n",
    "    Returns:\n",
    "    - A formatted DataFrame with combined metrics\n",
    "    \"\"\"\n",
    "    datasets = error_reduction_df['dataset'].unique()\n",
    "    \n",
    "    def process_df(df, metric_name, agg_method='median'):\n",
    "        df = df.copy()\n",
    "        df['prompt'] = df['type'].str.split('_').str[0]\n",
    "        df['model'] = df['type'].str.split('_').str[-1]\n",
    "        \n",
    "        if agg_method == 'median':\n",
    "            return df.groupby(['type', 'prompt', 'model', 'dataset'])[metric_name].median().reset_index()\n",
    "        elif agg_method == 'mean':\n",
    "            return df.groupby(['type', 'prompt', 'model', 'dataset'])[metric_name].mean().reset_index()\n",
    "        else:\n",
    "            raise ValueError(\"agg_method must be either 'median' or 'mean'\")\n",
    "    \n",
    "    # Process error reduction DataFrame\n",
    "    erp_df = process_df(error_reduction_df, 'erp', agg_method)\n",
    "    \n",
    "    # Process performance evaluation DataFrame\n",
    "    cer_df = process_df(performance_eval_df, 'cer', agg_method)\n",
    "    \n",
    "    # Rest of the function remains the same\n",
    "    combined_df = erp_df.merge(cer_df, on=['type', 'prompt', 'model', 'dataset'], suffixes=('_erp', '_cer'))\n",
    "    \n",
    "    combined_df = combined_df.pivot(index=['type', 'prompt', 'model'], columns='dataset', \n",
    "                                    values=['erp', 'cer']).round(2)\n",
    "    \n",
    "    combined_df.columns = [f'{metric.upper()}_{dataset}' for metric, dataset in combined_df.columns]\n",
    "    \n",
    "    combined_df = combined_df.reset_index()\n",
    "    \n",
    "    combined_df = combined_df.merge(model_name_code.reset_index().rename(columns={0: 'model'}), on='model')\n",
    "    combined_df = combined_df.rename(columns={'index': 'Model'})\n",
    "    \n",
    "    combined_df['Model'] = np.where(combined_df['prompt'] == 'boros', combined_df['Model'] + \" Boros\", combined_df['Model'])\n",
    "    \n",
    "    combined_df = combined_df.loc[(combined_df['prompt']!='instruct') & ~combined_df['Model'].str.contains('Llama 2')]\n",
    "    \n",
    "    combined_df = combined_df.set_index('Model').drop(columns=['type', 'prompt', 'model'])\n",
    "    \n",
    "    column_order = []\n",
    "    for dataset in datasets:\n",
    "        column_order.extend([f'ERP_{dataset}', f'CER_{dataset}'])\n",
    "    combined_df = combined_df[column_order]\n",
    "    \n",
    "    combined_df.columns = pd.MultiIndex.from_tuples([(col.split('_')[1], col.split('_')[0]) for col in combined_df.columns])\n",
    "    \n",
    "    combined_df = combined_df.sort_index(axis=1, level=0)\n",
    "    \n",
    "    return combined_df.reset_index()\n",
    "\n",
    "\n",
    "\n",
    "error_reduction_df = pd.concat([\n",
    "    ncse_error_reduction.assign(dataset='NCSE'),\n",
    "    smh_error_reduction.assign(dataset='SMH'),\n",
    "    ca_error_reduction.assign(dataset='CA')\n",
    "], ignore_index=True)\n",
    "\n",
    "# Create performance_eval_df\n",
    "performance_eval_df = pd.concat([\n",
    "    ncse_performance_eval.assign(dataset='NCSE'),\n",
    "    smh_performance_eval.assign(dataset='SMH'),\n",
    "    ca_performance_eval.assign(dataset='CA')\n",
    "], ignore_index=True)\n",
    "\n",
    "# Rename 'wer' column to 'erp' in error_reduction_df if necessary\n",
    "\n",
    "error_reduction_df = error_reduction_df.rename(columns={'CER': 'erp'})\n",
    "performance_eval_df = performance_eval_df.rename(columns={'CER': 'cer'})\n",
    "\n",
    "Original_error = {'NCSE':{'CER':0.17, 'ERP':0},\n",
    " 'SMH':{'CER':0.08, 'ERP':0},\n",
    " 'CA':{'CER':0.1, \"ERP\":0}}\n",
    "\n",
    "\n",
    "result_df = create_combined_metric_table(error_reduction_df, performance_eval_df, model_name_code, 'mean').sort_values(['Model'])\n",
    "\n",
    "result_df = result_df[['Model','NCSE', 'SMH', 'CA']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th colspan=\"2\" halign=\"left\">NCSE</th>\n",
       "      <th colspan=\"2\" halign=\"left\">SMH</th>\n",
       "      <th colspan=\"2\" halign=\"left\">CA</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>CER</th>\n",
       "      <th>ERP</th>\n",
       "      <th>CER</th>\n",
       "      <th>ERP</th>\n",
       "      <th>CER</th>\n",
       "      <th>ERP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Claude 3 Haiku</td>\n",
       "      <td>0.44</td>\n",
       "      <td>14.04</td>\n",
       "      <td>0.04</td>\n",
       "      <td>38.38</td>\n",
       "      <td>0.07</td>\n",
       "      <td>26.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Claude 3 Opus</td>\n",
       "      <td>0.07</td>\n",
       "      <td>64.09</td>\n",
       "      <td>0.04</td>\n",
       "      <td>51.03</td>\n",
       "      <td>0.05</td>\n",
       "      <td>48.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GPT-3.5</td>\n",
       "      <td>0.23</td>\n",
       "      <td>37.65</td>\n",
       "      <td>0.04</td>\n",
       "      <td>39.18</td>\n",
       "      <td>0.06</td>\n",
       "      <td>44.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>GPT-4</td>\n",
       "      <td>0.09</td>\n",
       "      <td>60.42</td>\n",
       "      <td>0.05</td>\n",
       "      <td>42.08</td>\n",
       "      <td>0.06</td>\n",
       "      <td>38.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GPT-4 Boros</td>\n",
       "      <td>0.09</td>\n",
       "      <td>61.67</td>\n",
       "      <td>0.04</td>\n",
       "      <td>48.44</td>\n",
       "      <td>0.05</td>\n",
       "      <td>45.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Gemma 7B</td>\n",
       "      <td>0.52</td>\n",
       "      <td>-2.34</td>\n",
       "      <td>0.17</td>\n",
       "      <td>-35.65</td>\n",
       "      <td>0.15</td>\n",
       "      <td>-38.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Llama 3</td>\n",
       "      <td>0.40</td>\n",
       "      <td>19.12</td>\n",
       "      <td>0.07</td>\n",
       "      <td>12.90</td>\n",
       "      <td>0.11</td>\n",
       "      <td>-9.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Mixtral 8x7B</td>\n",
       "      <td>0.36</td>\n",
       "      <td>6.60</td>\n",
       "      <td>0.10</td>\n",
       "      <td>-14.63</td>\n",
       "      <td>0.12</td>\n",
       "      <td>-16.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Overproof</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.05</td>\n",
       "      <td>28.38</td>\n",
       "      <td>0.07</td>\n",
       "      <td>34.59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Model  NCSE          SMH           CA       \n",
       "                    CER    ERP   CER    ERP   CER    ERP\n",
       "1  Claude 3 Haiku  0.44  14.04  0.04  38.38  0.07  26.01\n",
       "2   Claude 3 Opus  0.07  64.09  0.04  51.03  0.05  48.23\n",
       "4         GPT-3.5  0.23  37.65  0.04  39.18  0.06  44.22\n",
       "5           GPT-4  0.09  60.42  0.05  42.08  0.06  38.18\n",
       "0     GPT-4 Boros  0.09  61.67  0.04  48.44  0.05  45.55\n",
       "3        Gemma 7B  0.52  -2.34  0.17 -35.65  0.15 -38.01\n",
       "6         Llama 3  0.40  19.12  0.07  12.90  0.11  -9.42\n",
       "7    Mixtral 8x7B  0.36   6.60  0.10 -14.63  0.12 -16.30\n",
       "8       Overproof   NaN    NaN  0.05  28.38  0.07  34.59"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th colspan=\"2\" halign=\"left\">NCSE</th>\n",
       "      <th colspan=\"2\" halign=\"left\">SMH</th>\n",
       "      <th colspan=\"2\" halign=\"left\">CA</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>CER</th>\n",
       "      <th>ERP</th>\n",
       "      <th>CER</th>\n",
       "      <th>ERP</th>\n",
       "      <th>CER</th>\n",
       "      <th>ERP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Claude 3 Haiku</td>\n",
       "      <td>0.40</td>\n",
       "      <td>-237.41</td>\n",
       "      <td>0.11</td>\n",
       "      <td>-40.65</td>\n",
       "      <td>0.18</td>\n",
       "      <td>-92.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Claude 3 Opus</td>\n",
       "      <td>0.27</td>\n",
       "      <td>-28.51</td>\n",
       "      <td>0.05</td>\n",
       "      <td>45.92</td>\n",
       "      <td>0.06</td>\n",
       "      <td>43.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GPT-3.5</td>\n",
       "      <td>0.27</td>\n",
       "      <td>-42.03</td>\n",
       "      <td>0.09</td>\n",
       "      <td>-0.52</td>\n",
       "      <td>0.11</td>\n",
       "      <td>-18.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>GPT-4</td>\n",
       "      <td>0.17</td>\n",
       "      <td>58.03</td>\n",
       "      <td>0.06</td>\n",
       "      <td>32.65</td>\n",
       "      <td>0.07</td>\n",
       "      <td>29.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GPT-4 Boros</td>\n",
       "      <td>0.17</td>\n",
       "      <td>60.32</td>\n",
       "      <td>0.05</td>\n",
       "      <td>44.01</td>\n",
       "      <td>0.06</td>\n",
       "      <td>44.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Gemma 7B</td>\n",
       "      <td>0.46</td>\n",
       "      <td>-241.18</td>\n",
       "      <td>0.27</td>\n",
       "      <td>-227.08</td>\n",
       "      <td>0.29</td>\n",
       "      <td>-209.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Llama 3</td>\n",
       "      <td>0.36</td>\n",
       "      <td>-180.80</td>\n",
       "      <td>0.18</td>\n",
       "      <td>-111.94</td>\n",
       "      <td>0.20</td>\n",
       "      <td>-108.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Mixtral 8x7B</td>\n",
       "      <td>0.37</td>\n",
       "      <td>-113.22</td>\n",
       "      <td>0.17</td>\n",
       "      <td>-58.17</td>\n",
       "      <td>0.17</td>\n",
       "      <td>-60.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Overproof</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.07</td>\n",
       "      <td>27.28</td>\n",
       "      <td>0.07</td>\n",
       "      <td>34.11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Model  NCSE           SMH            CA        \n",
       "                    CER     ERP   CER     ERP   CER     ERP\n",
       "1  Claude 3 Haiku  0.40 -237.41  0.11  -40.65  0.18  -92.14\n",
       "2   Claude 3 Opus  0.27  -28.51  0.05   45.92  0.06   43.51\n",
       "4         GPT-3.5  0.27  -42.03  0.09   -0.52  0.11  -18.29\n",
       "5           GPT-4  0.17   58.03  0.06   32.65  0.07   29.70\n",
       "0     GPT-4 Boros  0.17   60.32  0.05   44.01  0.06   44.23\n",
       "3        Gemma 7B  0.46 -241.18  0.27 -227.08  0.29 -209.20\n",
       "6         Llama 3  0.36 -180.80  0.18 -111.94  0.20 -108.12\n",
       "7    Mixtral 8x7B  0.37 -113.22  0.17  -58.17  0.17  -60.65\n",
       "8       Overproof   NaN     NaN  0.07   27.28  0.07   34.11"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{table}\n",
      "\\caption{Model performance across the datasets measured in Error Reduction Percentage, higher is better.There is significant variation in how well the LMs are able to perform post-OCR correction, and significant differences between prompts for certain models.}\n",
      "\\label{tab:results}\n",
      "\\begin{tabular}{clrrrrrr}\n",
      "\\toprule\n",
      "Model & \\multicolumn{2}{r}{NCSE} & \\multicolumn{2}{r}{SMH} & \\multicolumn{2}{r}{CA} \\\\\n",
      " & CER & ERP & CER & ERP & CER & ERP \\\\\n",
      "\\midrule\n",
      "Claude 3 Haiku & 0.44 & 14.04 & 0.04 & 38.38 & 0.07 & 26.01 \\\\\n",
      "Claude 3 Opus & 0.07 & 64.09 & 0.04 & 51.03 & 0.05 & 48.23 \\\\\n",
      "GPT-3.5 & 0.23 & 37.65 & 0.04 & 39.18 & 0.06 & 44.22 \\\\\n",
      "GPT-4 & 0.09 & 60.42 & 0.05 & 42.08 & 0.06 & 38.18 \\\\\n",
      "GPT-4 Boros & 0.09 & 61.67 & 0.04 & 48.44 & 0.05 & 45.55 \\\\\n",
      "Gemma 7B & 0.52 & -2.34 & 0.17 & -35.65 & 0.15 & -38.01 \\\\\n",
      "Llama 3 & 0.40 & 19.12 & 0.07 & 12.90 & 0.11 & -9.42 \\\\\n",
      "Mixtral 8x7B & 0.36 & 6.60 & 0.10 & -14.63 & 0.12 & -16.30 \\\\\n",
      "Overproof & NaN & NaN & 0.05 & 28.38 & 0.07 & 34.59 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\\end{table}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Generate the LaTeX code for the DataFrame including the index\n",
    "combined_df = result_df\n",
    "latex_code = combined_df.to_latex(index=False, \n",
    "                                  float_format=\"%.2f\",\n",
    "                                  caption='Model performance across the datasets measured in Error Reduction Percentage, higher is better.There is significant variation in how well the LMs are able to perform post-OCR correction, and significant differences between prompts for certain models.',\n",
    "                                  label='tab:results')\n",
    "\n",
    "# Modify the LaTeX code to center column titles and ensure index is included\n",
    "# Replace '\\begin{tabular}{' with '\\begin{tabular}{' to center the columns\n",
    "latex_code = latex_code.replace(r'\\begin{tabular}{', r'\\begin{tabular}{c')  # Add column alignment for centering\n",
    "latex_code = latex_code.replace(r'c', r'c', 1)  # Ensure the first 'c' aligns with the first column header\n",
    "\n",
    "# Print the modified LaTeX code\n",
    "print(latex_code)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{table}\n",
      "\\caption{Model performance across the datasets measured in Error Reduction Percentage, higher is better.There is significant variation in how well the LMs are able to perform post-OCR correction, and significant differences between prompts for certain models.}\n",
      "\\label{tab:results}\n",
      "\\begin{tabular}{p{5cm}cccc}\n",
      "\\toprule\n",
      "Model & NCSE & SMH & CA \\\\\n",
      "\\midrule\n",
      "Claude 3 Haiku & 27.80 & 35.70 & 34.70 \\\\\n",
      "Claude 3 Opus & 62.70 & 45.50 & 47.00 \\\\\n",
      "GPT-3.5 & 39.40 & 42.90 & 44.10 \\\\\n",
      "GPT-4 & 59.80 & 41.80 & 37.60 \\\\\n",
      "Gemma 7B & 0.10 & -12.90 & -41.00 \\\\\n",
      "Llama 2 70B & -11.20 & 6.50 & -6.50 \\\\\n",
      "Llama 3 & 16.10 & 17.90 & 14.20 \\\\\n",
      "Mixtral 8x7B & 7.00 & -19.10 & -22.10 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\\end{table}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "results_tab = render_latex_with_formatting(result_df.loc[result_df['prompt']=='instruct', ['Model',  'NCSE', 'SMH', 'CA']].sort_values(['Model']), \n",
    "                             'Model performance across the datasets measured in Error Reduction Percentage: higher numbers indicate better performance, negative nummbers indicate OCR quality worse than baseline. There is significant variation in how well the LMs are able to perform post-OCR correction, and significant differences between prompts for certain models.',\n",
    "                             'tab:results')\n",
    "\n",
    "print(results_tab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>NCSE</th>\n",
       "      <th>SMH</th>\n",
       "      <th>CA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Claude 3 Haiku</td>\n",
       "      <td>27.8</td>\n",
       "      <td>35.7</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Claude 3 Opus</td>\n",
       "      <td>62.7</td>\n",
       "      <td>45.5</td>\n",
       "      <td>47.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>GPT-3.5</td>\n",
       "      <td>39.4</td>\n",
       "      <td>42.9</td>\n",
       "      <td>44.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>GPT-4</td>\n",
       "      <td>59.8</td>\n",
       "      <td>41.8</td>\n",
       "      <td>37.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Gemma 7B</td>\n",
       "      <td>0.1</td>\n",
       "      <td>-12.9</td>\n",
       "      <td>-41.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Llama 2 70B</td>\n",
       "      <td>-11.2</td>\n",
       "      <td>6.5</td>\n",
       "      <td>-6.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Llama 3</td>\n",
       "      <td>16.1</td>\n",
       "      <td>17.9</td>\n",
       "      <td>14.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Mixtral 8x7B</td>\n",
       "      <td>7.0</td>\n",
       "      <td>-19.1</td>\n",
       "      <td>-22.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Model  NCSE   SMH    CA\n",
       "9   Claude 3 Haiku  27.8  35.7  34.7\n",
       "15   Claude 3 Opus  62.7  45.5  47.0\n",
       "11         GPT-3.5  39.4  42.9  44.1\n",
       "12           GPT-4  59.8  41.8  37.6\n",
       "3         Gemma 7B   0.1 -12.9 -41.0\n",
       "1      Llama 2 70B -11.2   6.5  -6.5\n",
       "7          Llama 3  16.1  17.9  14.2\n",
       "5     Mixtral 8x7B   7.0 -19.1 -22.1"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.loc[result_df['prompt']=='instruct', ['Model',  'NCSE', 'SMH', 'CA']].sort_values(['Model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_melted = result_df.melt(id_vars=['model', 'prompt'], value_vars=['NCSE', 'SMH', 'CA'], \n",
    "                    var_name='Dataset', value_name='Value')\n",
    "\n",
    "df_full = result_df[result_df['prompt'] == 'full']\n",
    "df_instruct = result_df[result_df['prompt'] == 'instruct']\n",
    "\n",
    "# Melt the filtered DataFrames\n",
    "df_full_melted = df_full.melt(id_vars=['model'], value_vars=['NCSE', 'SMH', 'CA'], var_name='Dataset', value_name='Value_Full')\n",
    "df_instruct_melted = df_instruct.melt(id_vars=['model'], value_vars=['NCSE', 'SMH', 'CA'], var_name='Dataset', value_name='Value_Instruct')\n",
    "\n",
    "df_merged = pd.merge(df_full_melted, df_instruct_melted, on=['model', 'Dataset'])\n",
    "\n",
    "# Calculate the difference\n",
    "df_merged['Difference'] = df_merged['Value_Full'] - df_merged['Value_Instruct']\n",
    "\n",
    "df_merged = df_merged.loc[~(df_merged['model'].str.contains('llama2')),:].merge(model_name_code.reset_index().rename(columns={0: 'model'}), on='model').rename(columns={'index': 'Model'})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{table}\n",
      "\\caption{The differnce between the two prompts across all models and datasets}\n",
      "\\label{tab:prompt_diff}\n",
      "\\begin{tabular}{p{5cm}cccc}\n",
      "\\toprule\n",
      "Model & Dataset & Value_Full & Value_Instruct & Difference \\\\\n",
      "\\midrule\n",
      "Claude 3 Haiku & NCSE & 14.00 & 27.80 & -13.80 \\\\\n",
      "Claude 3 Haiku & CA & 26.00 & 34.70 & -8.70 \\\\\n",
      "Claude 3 Haiku & SMH & 38.40 & 35.70 & 2.70 \\\\\n",
      "Claude 3 Opus & CA & 48.20 & 47.00 & 1.20 \\\\\n",
      "Claude 3 Opus & NCSE & 64.10 & 62.70 & 1.40 \\\\\n",
      "Claude 3 Opus & SMH & 51.00 & 45.50 & 5.50 \\\\\n",
      "GPT-3.5 & CA & 44.20 & 44.10 & 0.10 \\\\\n",
      "GPT-3.5 & NCSE & 37.70 & 39.40 & -1.70 \\\\\n",
      "GPT-3.5 & SMH & 39.20 & 42.90 & -3.70 \\\\\n",
      "GPT-4 & NCSE & 60.40 & 59.80 & 0.60 \\\\\n",
      "GPT-4 & CA & 38.20 & 37.60 & 0.60 \\\\\n",
      "GPT-4 & SMH & 42.10 & 41.80 & 0.30 \\\\\n",
      "Gemma 7B & CA & -38.00 & -41.00 & 3.00 \\\\\n",
      "Gemma 7B & NCSE & -2.30 & 0.10 & -2.40 \\\\\n",
      "Gemma 7B & SMH & -35.70 & -12.90 & -22.80 \\\\\n",
      "Llama 3 & NCSE & 19.10 & 16.10 & 3.00 \\\\\n",
      "Llama 3 & CA & -9.40 & 14.20 & -23.60 \\\\\n",
      "Llama 3 & SMH & 12.90 & 17.90 & -5.00 \\\\\n",
      "Mixtral 8x7B & CA & -16.30 & -22.10 & 5.80 \\\\\n",
      "Mixtral 8x7B & NCSE & 6.60 & 7.00 & -0.40 \\\\\n",
      "Mixtral 8x7B & SMH & -14.60 & -19.10 & 4.50 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\\end{table}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "diff_df = df_merged.loc[:, ['Model', 'Dataset' ,'Value_Full', 'Value_Instruct', 'Difference']].sort_values('Model')\n",
    "\n",
    "print(render_latex_with_formatting(diff_df, caption = 'The difference between the two prompts across all models and datasets', label= 'tab:prompt_diff'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>Value_Full</th>\n",
       "      <th>Value_Instruct</th>\n",
       "      <th>Difference</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Claude 3 Haiku</td>\n",
       "      <td>NCSE</td>\n",
       "      <td>14.0</td>\n",
       "      <td>27.8</td>\n",
       "      <td>-13.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Claude 3 Haiku</td>\n",
       "      <td>CA</td>\n",
       "      <td>26.0</td>\n",
       "      <td>34.7</td>\n",
       "      <td>-8.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Claude 3 Haiku</td>\n",
       "      <td>SMH</td>\n",
       "      <td>38.4</td>\n",
       "      <td>35.7</td>\n",
       "      <td>2.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Claude 3 Opus</td>\n",
       "      <td>CA</td>\n",
       "      <td>48.2</td>\n",
       "      <td>47.0</td>\n",
       "      <td>1.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Claude 3 Opus</td>\n",
       "      <td>NCSE</td>\n",
       "      <td>64.1</td>\n",
       "      <td>62.7</td>\n",
       "      <td>1.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Claude 3 Opus</td>\n",
       "      <td>SMH</td>\n",
       "      <td>51.0</td>\n",
       "      <td>45.5</td>\n",
       "      <td>5.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>GPT-3.5</td>\n",
       "      <td>CA</td>\n",
       "      <td>44.2</td>\n",
       "      <td>44.1</td>\n",
       "      <td>0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GPT-3.5</td>\n",
       "      <td>NCSE</td>\n",
       "      <td>37.7</td>\n",
       "      <td>39.4</td>\n",
       "      <td>-1.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>GPT-3.5</td>\n",
       "      <td>SMH</td>\n",
       "      <td>39.2</td>\n",
       "      <td>42.9</td>\n",
       "      <td>-3.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>GPT-4</td>\n",
       "      <td>NCSE</td>\n",
       "      <td>60.4</td>\n",
       "      <td>59.8</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>GPT-4</td>\n",
       "      <td>CA</td>\n",
       "      <td>38.2</td>\n",
       "      <td>37.6</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>GPT-4</td>\n",
       "      <td>SMH</td>\n",
       "      <td>42.1</td>\n",
       "      <td>41.8</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Gemma 7B</td>\n",
       "      <td>CA</td>\n",
       "      <td>-38.0</td>\n",
       "      <td>-41.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Gemma 7B</td>\n",
       "      <td>NCSE</td>\n",
       "      <td>-2.3</td>\n",
       "      <td>0.1</td>\n",
       "      <td>-2.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Gemma 7B</td>\n",
       "      <td>SMH</td>\n",
       "      <td>-35.7</td>\n",
       "      <td>-12.9</td>\n",
       "      <td>-22.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Llama 3</td>\n",
       "      <td>NCSE</td>\n",
       "      <td>19.1</td>\n",
       "      <td>16.1</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Llama 3</td>\n",
       "      <td>CA</td>\n",
       "      <td>-9.4</td>\n",
       "      <td>14.2</td>\n",
       "      <td>-23.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Llama 3</td>\n",
       "      <td>SMH</td>\n",
       "      <td>12.9</td>\n",
       "      <td>17.9</td>\n",
       "      <td>-5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Mixtral 8x7B</td>\n",
       "      <td>CA</td>\n",
       "      <td>-16.3</td>\n",
       "      <td>-22.1</td>\n",
       "      <td>5.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Mixtral 8x7B</td>\n",
       "      <td>NCSE</td>\n",
       "      <td>6.6</td>\n",
       "      <td>7.0</td>\n",
       "      <td>-0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Mixtral 8x7B</td>\n",
       "      <td>SMH</td>\n",
       "      <td>-14.6</td>\n",
       "      <td>-19.1</td>\n",
       "      <td>4.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Model Dataset  Value_Full  Value_Instruct  Difference\n",
       "2   Claude 3 Haiku    NCSE        14.0            27.8       -13.8\n",
       "16  Claude 3 Haiku      CA        26.0            34.7        -8.7\n",
       "9   Claude 3 Haiku     SMH        38.4            35.7         2.7\n",
       "20   Claude 3 Opus      CA        48.2            47.0         1.2\n",
       "6    Claude 3 Opus    NCSE        64.1            62.7         1.4\n",
       "13   Claude 3 Opus     SMH        51.0            45.5         5.5\n",
       "18         GPT-3.5      CA        44.2            44.1         0.1\n",
       "4          GPT-3.5    NCSE        37.7            39.4        -1.7\n",
       "11         GPT-3.5     SMH        39.2            42.9        -3.7\n",
       "5            GPT-4    NCSE        60.4            59.8         0.6\n",
       "19           GPT-4      CA        38.2            37.6         0.6\n",
       "12           GPT-4     SMH        42.1            41.8         0.3\n",
       "14        Gemma 7B      CA       -38.0           -41.0         3.0\n",
       "0         Gemma 7B    NCSE        -2.3             0.1        -2.4\n",
       "7         Gemma 7B     SMH       -35.7           -12.9       -22.8\n",
       "3          Llama 3    NCSE        19.1            16.1         3.0\n",
       "17         Llama 3      CA        -9.4            14.2       -23.6\n",
       "10         Llama 3     SMH        12.9            17.9        -5.0\n",
       "15    Mixtral 8x7B      CA       -16.3           -22.1         5.8\n",
       "1     Mixtral 8x7B    NCSE         6.6             7.0        -0.4\n",
       "8     Mixtral 8x7B     SMH       -14.6           -19.1         4.5"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diff_df.sort_values('Value_Full')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj4AAAHcCAYAAADWVqC4AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB8V0lEQVR4nO3dd1gU1/s28HtpS1mKKAgiSkdUbFiigGAFO8YSFQvW2KOxx6jYe+9JFGyxl/i1d6NiV7CBAoLYsAuCAQTm/cOX+bksIOAiZe/Pde11sWfOnHlmdmAfzpyZIxEEQQARERGRClAr7ACIiIiIvhcmPkRERKQymPgQERGRymDiQ0RERCqDiQ8RERGpDCY+REREpDKY+BAREZHKYOJDREREKoOJDxEREakMJj70Vf7+/pBIJHJlqampGDt2LCwtLaGmpgYfHx8AQEJCAvr16wczMzNIJBKMGDHi+wdcRERHR0MikWDBggWFHQr9fxmfSWBgoFiW1fld1BSHGImKCyY+KiYwMBASiUR8aWtro1y5cvDy8sKyZcvw4cOHXLWzfv16zJ8/Hx07dsSGDRswcuRIAMCsWbMQGBiIQYMGYdOmTejRo0dB7o7KuXfvHvz9/REdHV3YoRQYKysr+Pv7i+/PnDkjd85++erSpct3iaG48PPzg6enp1yZRCLB0KFDC2ybs2bNwr59+wqs/W+JIePvnar6+++/sWTJksIOo8jRKOwAqHBMmzYN1tbW+PTpE2JjY3HmzBmMGDECixYtwv79+1GtWjWx7u+//47x48fLrX/q1ClYWFhg8eLFCuU//PADpkyZ8l32Q9Xcu3cPU6dOhaenJ6ysrAo7nO9q+PDhqFOnjlyZqh2DomjWrFno2LGj2OurqjEURX///Tfu3Lmj0j3vWWHio6JatGiB2rVri+8nTJiAU6dOoXXr1mjbti1CQ0Oho6MDANDQ0ICGhvyp8vLlSxgZGSm0+/LlS1SuXFlpcaanpyMlJQXa2tpKa5OKJ3d3d3Ts2LGww6BvkJiYCD09vcIOQ6lSU1ORnp4OLS2twg6FcomXukjUuHFjTJo0CY8ePcLmzZvF8i/HF2SMkTh9+jTu3r0rXnLIuBwRFRWFgwcPiuUZl2SSk5MxZcoU2NnZQSqVwtLSEmPHjkVycrJcDBnd8lu2bEGVKlUglUpx5MgRAMDTp0/Rp08flC1bFlKpFFWqVMH69evl1s+IY8eOHZg5cybKly8PbW1tNGnSBBEREQr7fPnyZbRs2RKlSpWCnp4eqlWrhqVLl8rVCQsLQ8eOHWFsbAxtbW3Url0b+/fvz9OxXbx4MSpWrAgdHR14eHjgzp07CnW+tp3AwEB06tQJANCoUSO5Y//rr7+idOnSEARBrD9s2DBIJBIsW7ZMLHvx4gUkEglWr14tluX2swGAzZs3w8XFBTo6OjA2NkaXLl3w+PFjuTqenp6oWrUq7t27h0aNGkFXVxcWFhaYN29eno5ZXlhZWcHPz0+h3NPTU+HSjzItWLAADRo0QOnSpaGjowMXFxfs2rVLoV7Geb1v3z5UrVpVPH8zzu0vnT9/HnXq1IG2tjZsbW2xdu1apcacl9+R8PBwdOjQAWZmZtDW1kb58uXRpUsXxMXFifuVmJiIDRs2iOdjxueQ8Xfj3r176NatG0qVKgU3NzcA2X8ufn5+Cr146enpWLp0KZydnaGtrQ0TExN4e3vj2rVrX40ht/z8/CCTyfDw4UN4eXlBT08P5cqVw7Rp0+R+p74ct7dkyRLY2tpCKpXi3r17AD73eLu7u0NPTw9GRkZo164dQkND5baVcVwePHiA7t27w9DQECYmJpg0aRIEQcDjx4/Rrl07GBgYwMzMDAsXLszy89u+fTt+++03mJmZQU9PD23btpX7XfT09MTBgwfx6NEj8biwh/Qz9viQnB49euC3337DsWPH0L9/f4XlJiYm2LRpE2bOnImEhATMnj0bAODk5IRNmzZh5MiRKF++PEaNGiXWT09PR9u2bXH+/HkMGDAATk5OuH37NhYvXowHDx4oXJs/deoUduzYgaFDh6JMmTKwsrLCixcv8MMPP4hfICYmJjh8+DD69u2L+Ph4ha7cOXPmQE1NDaNHj0ZcXBzmzZsHX19fXL58Waxz/PhxtG7dGubm5vjll19gZmaG0NBQHDhwAL/88gsA4O7du3B1dYWFhQXGjx8PPT097NixAz4+Pti9ezfat2//1WO6ceNGfPjwAUOGDEFSUhKWLl2Kxo0b4/bt2yhbtmyut9OwYUMMHz4cy5Ytw2+//QYnJyfx2L979w6LFy/G3bt3UbVqVQDAuXPnoKamhnPnzmH48OFiGQA0bNgQAPL02cycOROTJk1C586d0a9fP7x69QrLly9Hw4YNcfPmTbkewHfv3sHb2xs//vgjOnfujF27dmHcuHFwdnZGixYtvnrMsvLhwwe8fv1arszY2BhqaoX3/9vSpUvRtm1b+Pr6IiUlBdu2bUOnTp1w4MABtGrVSq7u+fPnsWfPHgwePBj6+vpYtmwZOnTogJiYGJQuXRoAcPv2bTRv3hwmJibw9/dHamoqpkyZIp4nyvS135GUlBR4eXkhOTkZw4YNg5mZGZ4+fYoDBw7g/fv3MDQ0xKZNm9CvXz/UrVsXAwYMAADY2trKbadTp06wt7fHrFmz5JKI3Orbty8CAwPRokUL9OvXD6mpqTh37hwuXbqE2rVr5yqG3EhLS4O3tzd++OEHzJs3D0eOHMGUKVOQmpqKadOmydUNCAhAUlISBgwYAKlUCmNjY5w4cQItWrSAjY0N/P398d9//2H58uVwdXXFjRs3FJKOn376CU5OTpgzZw4OHjyIGTNmwNjYGGvXrkXjxo0xd+5cbNmyBaNHj0adOnXE39kMM2fOhEQiwbhx4/Dy5UssWbIETZs2RXBwMHR0dDBx4kTExcXhyZMn4pAEmUyW5+NSIgmkUgICAgQAwtWrV7OtY2hoKNSsWVN8P2XKFCHzqeLh4SFUqVJFYd2KFSsKrVq1kivbtGmToKamJpw7d06ufM2aNQIA4cKFC2IZAEFNTU24e/euXN2+ffsK5ubmwuvXr+XKu3TpIhgaGgofP34UBEEQTp8+LQAQnJychOTkZLHe0qVLBQDC7du3BUEQhNTUVMHa2lqoWLGi8O7dO7k209PTxZ+bNGkiODs7C0lJSXLLGzRoINjb2yvs/5eioqIEAIKOjo7w5MkTsfzy5csCAGHkyJF53s7OnTsFAMLp06fltvXy5UsBgLBq1SpBEATh/fv3gpqamtCpUyehbNmyYr3hw4cLxsbG4j7m9rOJjo4W1NXVhZkzZ8rVu337tqChoSFX7uHhIQAQNm7cKJYlJycLZmZmQocOHXI8ZlnJ+EyzekVFRQmC8Pm869Wrl8K6Hh4egoeHh/g+4zMJCAgQy7I6v3Mr47zLkJKSIlStWlVo3LixXDkAQUtLS4iIiBDLQkJCBADC8uXLxTIfHx9BW1tbePTokVh27949QV1dPd8xAhCGDBkivs/t78jNmzcFAMLOnTtzbF9PTy/LY59xXLt27aqwLPPnkqFXr15CxYoVxfenTp0SAAjDhw9XqPvl72l2MeRWr169BADCsGHD5Npv1aqVoKWlJbx69UoQhP87fwwMDISXL1/KtVGjRg3B1NRUePPmjVgWEhIiqKmpCT179hTLMo7LgAEDxLLU1FShfPnygkQiEebMmSOWv3v3TtDR0ZHbt4zPz8LCQoiPjxfLd+zYIQAQli5dKpa1atVK7njSZ7zURQpkMlmu7+7KjZ07d8LJyQmVKlXC69evxVfjxo0BAKdPn5ar7+HhITdOSBAE7N69G23atIEgCHJteHl5IS4uDjdu3JBro3fv3nLX3N3d3QEADx8+BADcvHkTUVFRGDFihMJYpYzLem/fvsWpU6fQuXNnsbfh9evXePPmDby8vBAeHo6nT59+df99fHxgYWEhvq9bty7q1auHQ4cOKW07JiYmqFSpEv79918AwIULF6Curo4xY8bgxYsXCA8PB/C5x8fNzU3cx9x+Nnv27EF6ejo6d+4sV8/MzAz29vYKn6FMJkP37t3F91paWqhbt654/PNj8uTJOH78uNzLzMws3+0pQ8Y4OOBzL1dcXBzc3d0VzkcAaNq0qVxPRLVq1WBgYCAek7S0NBw9ehQ+Pj6oUKGCWM/JyQleXl5Kj/1rvyOGhoYAgKNHj+Ljx4/53s7AgQPzve7u3bshkUiyvFmiIO7W+vLut4ze5ZSUFJw4cUKuXocOHWBiYiK+f/78OYKDg+Hn5wdjY2OxvFq1amjWrJn4u/6lfv36iT+rq6ujdu3aEAQBffv2FcuNjIzg6OiY5e9Nz549oa+vL77v2LEjzM3Ns9wWyeOlLlKQkJAAU1NTpbUXHh6O0NBQuT8UX3r58qXce2tra7n3r169wvv37/HHH3/gjz/+yFUbX35xAECpUqUAfP5yAoDIyEgAEC8LZSUiIgKCIGDSpEmYNGlSttv9MqnJir29vUKZg4MDduzYodTtuLu7i3/0zp07h9q1a6N27dowNjbGuXPnULZsWYSEhKBbt27iOrn9bMLDwyEIQpb7AgCamppy78uXL6/wxVSqVCncunUrx33IibOzM5o2bZrv9QvCgQMHMGPGDAQHB8uNicrqSznzOQl8PiYZ5+SrV6/w33//ZXmMHR0dlf6F9rXfEWtra/z6669YtGgRtmzZAnd3d7Rt21Ycl5JbmX+f8yIyMhLlypWTSyYKipqaGmxsbOTKHBwcAEDh8RGZ9+nRo0cAPn9OmTk5OeHo0aMKA7szH39DQ0Noa2ujTJkyCuVv3rxRaDfzeSKRSGBnZ1eiH3WhLEx8SM6TJ08QFxcHOzs7pbWZnp4OZ2dnLFq0KMvllpaWcu+//C86Y30A6N69O3r16pVlG1/efg98/g8qK0IexhhkbHf06NHZ/setjOOkrO24ubnhzz//xMOHD3Hu3Dm4u7tDIpHAzc0N586dQ7ly5ZCeni7+Z5+x7dx8Nunp6ZBIJDh8+HCWxzbz2AFlHP+8yO6//7S0tGxj+Vbnzp1D27Zt0bBhQ6xatQrm5ubQ1NREQEAA/v77b4X63/uYfE1u4lm4cCH8/Pzwzz//4NixYxg+fDhmz56NS5cuoXz58rnaTubfZ+Dz55XVfqelpeUy+sKV1T7lVVbHv6idIyUVEx+Ss2nTJgBQate6ra0tQkJC0KRJk3x1T5uYmEBfXx9paWlK+48/45LDnTt3sm0z478/TU3Nb9puxmWmLz148EAc7JiX7eR0/DISmuPHj+Pq1avis5caNmyI1atXo1y5ctDT04OLi4u4Tm4/G1tbWwiCAGtra/G/4KKkVKlSeP/+vUL5o0ePFP6LV5bdu3dDW1sbR48ehVQqFcsDAgLy1Z6JiQl0dHSyPF/u37+f7zi/lbOzM5ydnfH7778jKCgIrq6uWLNmDWbMmAEgf5ecSpUqleXlm4yekwy2trY4evQo3r59m2OvjzIue6Wnp+Phw4dy5/eDBw8AfP15URUrVgSQ9ecUFhaGMmXKKP02/szniSAIiIiIkPsnUJUf3pgTjvEh0alTpzB9+nRYW1vD19dXae127twZT58+xZ9//qmw7L///kNiYmKO66urq6NDhw7YvXt3lreBv3r1Ks8x1apVC9bW1liyZInCF2bGf1empqbw9PTE2rVr8fz583xvd9++fXJjdK5cuYLLly+LdzflZTsZfzyz+pK3trYWHyr56dMnuLq6AvicEEVGRmLXrl344Ycf5J7JlNvP5scff4S6ujqmTp2q8N+nIAhZdsV/T7a2trh06RJSUlLEsgMHDijcaq9M6urqkEgkcr0U0dHR+X6Ksbq6Ory8vLBv3z7ExMSI5aGhoTh69Oi3hptn8fHxSE1NlStzdnaGmpqa3GU9PT29LM/HnNja2iIsLEzu3A4JCcGFCxfk6nXo0AGCIGDq1KkKbXx5HuYnhqysWLFCrv0VK1ZAU1MTTZo0yXE9c3Nz1KhRAxs2bJCL486dOzh27Bhatmz5zbFllnG3aIZdu3bh+fPncndN6unpiY8eoP/DHh8VdfjwYYSFhSE1NRUvXrzAqVOncPz4cVSsWBH79+9X6gMDe/TogR07dmDgwIE4ffo0XF1dkZaWhrCwMOzYsQNHjx6Ve5hiVubMmYPTp0+jXr166N+/PypXroy3b9/ixo0bOHHiBN6+fZunmNTU1LB69Wq0adMGNWrUQO/evWFubo6wsDDcvXtX/KJZuXIl3Nzc4OzsjP79+8PGxgYvXrzAxYsX8eTJE4SEhHx1W3Z2dnBzc8OgQYOQnJyMJUuWoHTp0hg7dqxYJ7fbqVGjBtTV1TF37lzExcVBKpWicePG4pgsd3d3bNu2Dc7OzuKYjVq1akFPTw8PHjyQG98D5P6zsbW1xYwZMzBhwgRER0fDx8cH+vr6iIqKwt69ezFgwACMHj06T5+BMvXr1w+7du2Ct7c3OnfujMjISGzevDlftzXnVqtWrbBo0SJ4e3ujW7duePnyJVauXAk7O7t8j2WaOnUqjhw5And3dwwePBipqalYvnw5qlSp8k3jo/Lj1KlTGDp0KDp16gQHBwekpqZi06ZN4j8iGVxcXHDixAksWrQI5cqVg7W1NerVq5dj23369MGiRYvg5eWFvn374uXLl1izZg2qVKmC+Ph4sV6jRo3Qo0cPLFu2DOHh4fD29kZ6ejrOnTuHRo0aiYOR8xNDZtra2jhy5Ah69eqFevXq4fDhwzh48CB+++23bMfAfWn+/Plo0aIF6tevj759+4q3sxsaGhbI9CfGxsZwc3ND79698eLFCyxZsgR2dnZyjyFxcXHB9u3b8euvv6JOnTqQyWRo06aN0mMpdr7vTWRU2DJuZ894aWlpCWZmZkKzZs2EpUuXyt0emeFbb2cXhM+3+c6dO1eoUqWKIJVKhVKlSgkuLi7C1KlThbi4OLEeMt16+6UXL14IQ4YMESwtLQVNTU3BzMxMaNKkifDHH3+IdTJu9cx8C25WtzELgiCcP39eaNasmaCvry/o6ekJ1apVk7u9WBAEITIyUujZs6dgZmYmaGpqChYWFkLr1q2FXbt2ZRln5m3Onz9fWLhwoWBpaSlIpVLB3d1dCAkJUaif2+38+eefgo2NjXiL85e3tq9cuVIAIAwaNEhunaZNmwoAhJMnTypsN7efjSAIwu7duwU3NzdBT09P0NPTEypVqiQMGTJEuH//vlgnu3Mj863KuZXdZ5rZwoULBQsLC0EqlQqurq7CtWvXCvx29nXr1gn29vaCVCoVKlWqJAQEBGTZXnbndVa34Z89e1ZwcXERtLS0BBsbG2HNmjXfFGPmbef2d+Thw4dCnz59BFtbW0FbW1swNjYWGjVqJJw4cUJuvbCwMKFhw4aCjo6OAEDcn4yYM24Fz2zz5s2CjY2NoKWlJdSoUUM4evRoludIamqqMH/+fKFSpUqClpaWYGJiIrRo0UK4fv36V2PIrV69egl6enpCZGSk0Lx5c0FXV1coW7asMGXKFCEtLU3hGM2fPz/Ldk6cOCG4uroKOjo6goGBgdCmTRvh3r17cnWyOy4ZMWSW+fcp4/PbunWrMGHCBMHU1FTQ0dERWrVqJfcYBEEQhISEBKFbt26CkZGRAIC3tv9/EkHgqCkiIlJdfn5+2LVrFxISEgo7lK86c+YMGjVqhJ07d3IKl3ziGB8iIiJSGUx8iIiISGUw8SEiIiKVwTE+REREpDLY40NEREQqg4kPERERqQwmPvRdBQYGQiKRcCI9JQoPD0fz5s1haGgIiUSS7ycHU+Gen35+fl+dGoGIvh0THyrWPn78CH9/f5w5c6awQyk0vXr1wu3btzFz5kxs2rQp26dgR0dHQyKRZPuaM2fOd448dzKSkYyXhoYGLCws4OfnJzcVSHHw7Nkz+Pv7Izg4uLBDEWU+L9TV1VGhQgW0b9++SMWZX/wbQZlxygr6rnr06IEuXbrITer4LT5+/CjO4+Pp6amUNouT//77DxcvXsTEiRPFx/d/TdeuXbOcO6hmzZrKDk+ppk2bBmtrayQlJeHSpUsIDAzE+fPncefOHaVOsVKQnj17hqlTp8LKygo1atSQW/bnn38iPT29cALD/50XaWlpCA0NxerVq3H48GFcunRJIdbiRNX/RpAiJj70Xamrq0NdXb2ww/iqxMREpc+mXBAyJnk0MjLK9Tq1atVC9+7d87QdQRCQlJQEHR0dhWVJSUnQ0tKCmlr+O5Bzc7xbtGgh9mb169cPZcqUwdy5c7F//3507tw539suKjQ1NQt1+5nPC1dXV7Rt2xarV6/G2rVrv6nt4vL7RKqBl7rou8pqDIWVlRVat26N8+fPo27dutDW1oaNjQ02btyYY1vR0dHi5IFTp04Vu+q/nBAwLCwMHTt2hLGxMbS1tVG7dm3s378/y5jOnj2LwYMHw9TUFOXLlwfw+T/EqlWr4tatW/Dw8ICuri7s7Oywa9cuAMDZs2dRr1496OjowNHRESdOnJBr+8OHDxgxYgSsrKwglUphamqKZs2a4caNG189Vjdv3kSLFi1gYGAAmUyGJk2a4NKlS+Jyf39/VKxYEQAwZswYSCQSpY0RyfhMMiYp1dHRwdq1a3HmzBlIJBJs27YNv//+OywsLKCrqytOLLlz5064uLhAR0cHZcqUQffu3RUuR/n5+UEmkyEyMhItW7aEvr4+fH198xyju7s7ACAyMlKuPDefOQDcvXsXjRs3ho6ODsqXL48ZM2Zk2eOS+Zz68hj5+fnJlb1//x4jR44UP+/y5cujZ8+eeP36Nc6cOYM6deoAAHr37i2er4GBgeJxyfz5JSYmYtSoUbC0tIRUKoWjoyMWLFiAzE8hkUgkGDp0KPbt24eqVatCKpWiSpUqOHLkSE6HMEeNGzcGAERFRYllly9fhre3NwwNDaGrqwsPDw+FGdX9/f0hkUhw7949dOvWDaVKlYKbm5u4fPPmzahbty50dXVRqlQpNGzYEMeOHZNr4/Dhw3B3d4eenh709fXRqlUr3L17V65Oxnn09OlT+Pj4QCaTwcTEBKNHj0ZaWhqAr/+NuHXrFvz8/GBjYwNtbW2YmZmhT58+ePPmjcLxOHPmDGrXrg1tbW3Y2tpi7dq14r5mtnnzZvH3wNjYGF26dMHjx49ze+ipgLHHh4qEiIgIdOzYEX379kWvXr2wfv16+Pn5wcXFBVWqVMlyHRMTE6xevRqDBg1C+/bt8eOPPwIAqlWrBuDzF5urqyssLCwwfvx46OnpYceOHfDx8cHu3bvRvn17ufYGDx4MExMTTJ48GYmJiWL5u3fv0Lp1a3Tp0gWdOnXC6tWr0aVLF2zZsgUjRozAwIED0a1bN8yfPx8dO3bE48ePoa+vDwAYOHAgdu3ahaFDh6Jy5cp48+YNzp8/j9DQUNSqVSvb43H37l24u7vDwMAAY8eOhaamJtauXQtPT08x2frxxx9hZGSEkSNHipcpZDLZV4/1x48f8fr1a4VyIyMjaGj835+E+/fvo2vXrvj555/Rv39/ODo6isumT58OLS0tjB49GsnJydDS0kJgYCB69+6NOnXqYPbs2Xjx4gWWLl2KCxcu4ObNm3K9UqmpqfDy8oKbmxsWLFgAXV3dr8adWUbynDELfcZxy81nHhsbi0aNGiE1NVWs98cff2TZo5VbCQkJcHd3R2hoKPr06YNatWrh9evX2L9/P548eQInJydMmzYNkydPxoABA8TErUGDBlm2JwgC2rZti9OnT6Nv376oUaMGjh49ijFjxuDp06dYvHixXP3z589jz549GDx4MPT19bFs2TJ06NABMTExKF26dJ73JyOhzFj31KlTaNGiBVxcXDBlyhSoqakhICAAjRs3xrlz51C3bl259Tt16gR7e3vMmjVLTNSmTp0Kf39/NGjQANOmTYOWlhYuX76MU6dOoXnz5gCATZs2oVevXvDy8sLcuXPx8eNHrF69Gm5ubrh586ZccpiWlgYvLy/Uq1cPCxYswIkTJ7Bw4ULY2tpi0KBBX/0bcfz4cTx8+BC9e/eGmZkZ7t69iz/++AN3797FpUuXxKTm5s2b8Pb2hrm5OaZOnYq0tDRMmzYty1nbZ86ciUmTJqFz587o168fXr16heXLl6Nhw4YKvwdUSApvflRSRRmzw0dFRYllFStWFAAI//77r1j28uVLQSqVCqNGjcqxvVevXgkAhClTpigsa9KkieDs7CwkJSWJZenp6UKDBg0Ee3t7hZjc3NyE1NRUuTY8PDwEAMLff/8tloWFhQkABDU1NeHSpUti+dGjRxVm/jY0NMx2tvmc+Pj4CFpaWkJkZKRY9uzZM0FfX19o2LChWPa12aK/lFE3u9fFixfFuhmfyZEjR+TayJgZ2sbGRvj48aNYnpKSIpiamgpVq1YV/vvvP7H8wIEDAgBh8uTJYlmvXr0EAML48eNzdSwyPp8TJ04Ir169Eh4/fizs2rVLMDExEaRSqfD48WOxbm4/8xEjRggAhMuXL4tlL1++FAwNDRXOz+zOr8wzq0+ePFkAIOzZs0ehbnp6uiAIgnD16lWFcyRD5pnJ9+3bJwAQZsyYIVevY8eOgkQiESIiIuRi1NLSkisLCQkRAAjLly9X2NaXMs6LqVOnCq9evRJiY2OFM2fOCDVr1hQACLt37xbS09MFe3t7wcvLS9wXQRCEjx8/CtbW1kKzZs3EsozZx7t27Sq3nfDwcEFNTU1o37693IznXx6fDx8+CEZGRkL//v3llsfGxgqGhoZy5Rnn0bRp0+Tq1qxZU3BxcRHf5/Q34stzOMPWrVsV/h61adNG0NXVFZ4+fSq3PxoaGsKXX6PR0dGCurq6MHPmTLk2b9++LWhoaCiUU+HgpS4qEipXriz+Bwx87s1xdHTEw4cP89Xe27dvcerUKXTu3BkfPnzA69ev8fr1a7x58wZeXl4IDw9XuATTv3//LMcfyWQydOnSRXzv6OgIIyMjODk5oV69emJ5xs9fxmxkZITLly/j2bNnuY49LS0Nx44dg4+PD2xsbMRyc3NzdOvWDefPnxcvLeXHgAEDcPz4cYVX5cqV5epZW1vDy8sryzZ69eol1zty7do1vHz5EoMHD5YbaNyqVStUqlQJBw8eVGhj0KBBeYq7adOmMDExgaWlJTp27Ag9PT3s379fvCyZl8/80KFD+OGHH+R6KUxMTPJ1yS3D7t27Ub16dYWeRABZXg75mkOHDkFdXR3Dhw+XKx81ahQEQcDhw4flyps2bQpbW1vxfbVq1WBgYJDr36EpU6bAxMQEZmZm8PT0RGRkJObOnYsff/wRwcHBCA8PR7du3fDmzRvx2CYmJqJJkyb4999/FS4TDhw4UO79vn37kJ6ejsmTJyuMB8s4PsePH8f79+/RtWtXcRuvX7+Guro66tWrh9OnTyvEnXk77u7uud7nL8/hpKQkvH79Gj/88AMAiJej09LScOLECfj4+KBcuXJifTs7O7Ro0UKuvT179iA9PR2dO3eWi9/MzAz29vZZxk/fHy91UZFQoUIFhbJSpUrh3bt3+WovIiICgiBg0qRJmDRpUpZ1Xr58CQsLC/G9tbV1lvXKly+v8MVlaGgIS0tLhTIAcjHPmzcPvXr1gqWlJVxcXNCyZUv07NlTLqHJ7NWrV/j48aPcpaUMTk5OSE9Px+PHj7O9BPg19vb2aNq06VfrZXc8slr26NEjAMgy5kqVKuH8+fNyZRoaGmLCklsrV66Eg4MD4uLisH79evz7779ydwfm5TN/9OiRXNKaIav4cysyMhIdOnTI9/qZPXr0COXKlRMvm2ZwcnISl3/pW3+HBgwYgE6dOkFNTQ1GRkaoUqWKeHzDw8MBfE54sxMXFyd32THzORIZGQk1NTWFBPtLGdvJGF+UmYGBgdx7bW1thctNednnt2/fYurUqdi2bRtevnwptywuLg7A53Pmv//+g52dncL6mcvCw8MhCALs7e2z3F5hD2Cnz5j4UJGQ3Z1eQj6nksv473P06NHZ9lpk/qOV3fiO7GLLTcydO3eGu7s79u7di2PHjmH+/PmYO3cu9uzZo/DfYlGT03iXbxkLAwBSqTTPd4HVrVtXvKvLx8cHbm5u6NatG+7fvw+ZTJavz/xbZAygLSq+9Xcop4Q449jOnz8/21vbM48vy885krGdTZs2wczMTGH5l2PQgOz3Obc6d+6MoKAgjBkzBjVq1BDPI29v73w9WiA9PR0SiQSHDx/OtveYCh8THyrWsruEkNGjoqmpmavejYJkbm6OwYMHY/DgwXj58iVq1aqFmTNnZpv4mJiYQFdXF/fv31dYFhYWBjU1NYXepsKWcXfZ/fv3Ff5bv3//vrhcWdTV1TF79mw0atQIK1aswPjx4/P0mVesWFHsXcgca2alSpXC+/fv5cpSUlLw/PlzuTJbW1vcuXMnx+3m5ZJXxYoVceLECXz48EGu1ycsLExc/r1kXEIzMDDI9++Tra0t0tPTce/evWyTp4ztmJqaKu33Nrtj/u7dO5w8eRJTp07F5MmTxfLM54WpqSm0tbURERGh0EbmMltbWwiCAGtrazg4OCgheioIHONDxVrG3UCZv5hMTU3h6emJtWvXKnxBAf/3/JuClJaWJnaXfxlXuXLlkJycnO166urqaN68Of755x+52/5fvHiBv//+G25ubgpd/oWtdu3aMDU1xZo1a+T27fDhwwgNDUWrVq2Uvk1PT0/UrVsXS5YsQVJSUp4+85YtW+LSpUu4cuWK3PItW7YorGdra4t///1XruyPP/5Q6PHp0KEDQkJCsHfvXoU2MnpdMp5lk/l8zUrGwwRXrFghV7548WJIJJLv2mPo4uICW1tbLFiwAAkJCQrLc/P75OPjAzU1NUybNk2hNyXj+Hh5ecHAwACzZs3Cp0+f8rWdzLL7G5HRI5O5R2zJkiUK9Zo2bYp9+/bJjdWLiIhQGGf1448/Ql1dHVOnTlVoVxCELG+Tp++PPT5UrOno6KBy5crYvn07HBwcYGxsjKpVq6Jq1apYuXIl3Nzc4OzsjP79+8PGxgYvXrzAxYsX8eTJE4SEhBRobB8+fED58uXRsWNHVK9eHTKZDCdOnMDVq1excOHCHNedMWMGjh8/Djc3NwwePBgaGhpYu3YtkpOTMW/evG+K68aNG9i8ebNCua2tLerXr5+vNjU1NTF37lz07t0bHh4e6Nq1q3g7u5WVFUaOHPlNMWdnzJgx6NSpEwIDAzFw4MBcf+Zjx47Fpk2b4O3tjV9++UW8nb1ixYq4deuW3Db69euHgQMHokOHDmjWrBlCQkJw9OhRlClTRiGWXbt2oVOnTujTpw9cXFzw9u1b7N+/H2vWrEH16tVha2sLIyMjrFmzBvr6+tDT00O9evWyHE/Vpk0bNGrUCBMnTkR0dDSqV6+OY8eO4Z9//sGIESPkBjIXNDU1Nfz1119o0aIFqlSpgt69e8PCwgJPnz7F6dOnYWBggP/97385tmFnZ4eJEydi+vTpcHd3x48//gipVIqrV6+iXLlymD17NgwMDLB69Wr06NEDtWrVQpcuXWBiYoKYmBgcPHgQrq6uCong1+T0N6Jhw4aYN28ePn36BAsLCxw7dkzuuUUZ/P39cezYMbi6umLQoEFiQlq1alW5aT1sbW0xY8YMTJgwAdHR0fDx8YG+vj6ioqKwd+9eDBgwAKNHj85T/FQACuVeMlJZ2d3O3qpVK4W6Hh4egoeHx1fbDAoKElxcXAQtLS2F21YjIyOFnj17CmZmZoKmpqZgYWEhtG7dWti1a5dCTFevXs0yhipVqiiUZxczAPH29eTkZGHMmDFC9erVBX19fUFPT0+oXr26sGrVqq/ukyAIwo0bNwQvLy9BJpMJurq6QqNGjYSgoCC5Osq8nf3LW7Oz27+M29l37tyZ5Ta2b98u1KxZU5BKpYKxsbHg6+srPHnyRK5Or169BD09vVwcgc9y+nzS0tIEW1tbwdbWVnwUQW4+c0EQhFu3bgkeHh6Ctra2YGFhIUyfPl1Yt26dwvmZlpYmjBs3TihTpoygq6sreHl5CREREQq3swuCILx580YYOnSoYGFhIWhpaQnly5cXevXqJbx+/Vqs888//wiVK1cWb4XOuLU98+3sgvD59u6RI0cK5cqVEzQ1NQV7e3th/vz5creUC4L8efelrGLMLC/n0M2bN4Uff/xRKF26tCCVSoWKFSsKnTt3Fk6ePCnWybid/dWrV1m2sX79evEcKVWqlODh4SEcP35crs7p06cFLy8vwdDQUNDW1hZsbW0FPz8/4dq1a2Kd7M6jjO1/Kbu/EU+ePBHat28vGBkZCYaGhkKnTp2EZ8+eZXn7+8mTJ4WaNWsKWlpagq2trfDXX38Jo0aNErS1tRVi2L17t+Dm5ibo6ekJenp6QqVKlYQhQ4YI9+/fz/H40vchEYR8jh4lIiJSYT4+Prh7926W48Wo6OIYHyIioq/477//5N6Hh4fj0KFDnPi0GGKPDxER0VeYm5uL83o9evQIq1evRnJyMm7evJntc3uoaOLgZiIioq/w9vbG1q1bERsbC6lUivr162PWrFlMeooh9vgQERGRyuAYHyIiIlIZTHyIiIhIZXCMTybp6el49uwZ9PX18zWjMhEREX1/giDgw4cPKFeuXI5zATLxyeTZs2dFbh4kIiIiyp3Hjx+jfPny2S5n4pNJxmSAjx8/LnLzIREREVHW4uPjYWlpKTepb1aY+GSScXnLwMCAiQ8REVEx87VhKhzcTERERCqDiQ8RERGpDCY+REREpDKY+BAREZHKYOJDREREKoOJDxEREakMJj5ERESkMpj4EBERkcpg4kNEREQqg4kPERERqQwmPkRERKQymPgQERGRymDiQ0RERCqDiQ8RERGpDI3CDoCIiOhbxUxzVlpbFSbfVlpbVPSwx4eIiIhUBhMfIiIiUhm81EVEVEyEzjyltLacJjZWWltExQl7fIiIiEhllKjEx9/fHxKJRO5VqVKlwg6LiIiIiogSd6mrSpUqOHHihPheQ6PE7SIRERHlU4nLCjQ0NGBmZlbYYRAREVERVKIudQFAeHg4ypUrBxsbG/j6+iImJibH+snJyYiPj5d7ERERUclUohKfevXqITAwEEeOHMHq1asRFRUFd3d3fPjwIdt1Zs+eDUNDQ/FlaWn5HSMmIiKi76lEJT4tWrRAp06dUK1aNXh5eeHQoUN4//49duzYke06EyZMQFxcnPh6/Pjxd4yYiIiIvqcSN8bnS0ZGRnBwcEBERES2daRSKaRS6XeMioiIiApLierxySwhIQGRkZEwNzcv7FCIiIioCChRic/o0aNx9uxZREdHIygoCO3bt4e6ujq6du1a2KERERFREVCiLnU9efIEXbt2xZs3b2BiYgI3NzdcunQJJiYmhR0aERERFQElKvHZtm1bYYdAVOLFTHNWWlsVJt9WWltERLlRoi51EREREeWEiQ8RERGpDCY+REREpDKY+BAREZHKYOJDREREKoOJDxEREakMJj5ERESkMpj4EBERkcpg4kNEREQqg4kPERERqQwmPkRERKQymPgQERGRymDiQ0RERCqDiQ8RERGpDCY+REREpDKY+BAREZHK0CjsAKh4OtvQQ2ltefx7VmltEVHx4TJmo9La2quvtKaohGOPDxEREakMJj5ERESkMnipi4gKjetyV6W1dWHYBaW1RUQlF3t8iIiISGUw8SEiIiKVwcSHiIiIVAYTHyIiIlIZTHyIiIhIZTDxISIiIpXBxIeIiIhUBhMfIiIiUhlMfIiIiEhlMPEhIiIilcHEh4iIiFQGEx8iIiJSGUx8iIiISGUw8SEiIiKVoVHYARAVRaEzTymtLaeJjZXWFhERfRv2+BAREZHKYI8PERER5Utx7B1njw8RERGpDCY+REREpDKY+BAREZHKYOJDREREKoOJDxEREakMJj5ERESkMng7O5UoM7t3VEo7PzoNVko7RERFjbL+TgLF828le3yIiIhIZTDxISIiIpVRIhOflStXwsrKCtra2qhXrx6uXLlS2CERERFREVDixvhs374dv/76K9asWYN69ephyZIl8PLywv3792FqalrY4RGRilH18RRERU2J6/FZtGgR+vfvj969e6Ny5cpYs2YNdHV1sX79+sIOjYiIiApZierxSUlJwfXr1zFhwgSxTE1NDU2bNsXFixezXCc5ORnJycni+/j4+AKPk4iIKC9WjPpfYYdQYpSoxOf169dIS0tD2bJl5crLli2LsLCwLNeZPXs2pk6dqtQ4XMZsVFpbe/XnK62trqUMlNbWhX8vKK0tZf5CT9y8S2ltlSTKPCevz7+ttLaUdxYBZxt6KK2t23VGK60tnpPZuz6/pxJbU15brstdldLOrJ3K+4od+u9ZpbUFtFFiW8VPibvUlVcTJkxAXFyc+Hr8+HFhh0REREQFpET1+JQpUwbq6up48eKFXPmLFy9gZmaW5TpSqRRSqfR7hEdERESFrET1+GhpacHFxQUnT54Uy9LT03Hy5EnUr1+/ECMjIiKioqBE9fgAwK+//opevXqhdu3aqFu3LpYsWYLExET07t27sEMjIiKiQlbiEp+ffvoJr169wuTJkxEbG4saNWrgyJEjCgOei4sKk5U3kBRKGrBHRERUXJW4xAcAhg4diqFDhxZ2GERERFTElKgxPkREREQ5YeJDREREKoOJDxEREakMJj5ERESkMpj4EBERkcpg4kNEREQqg4kPERERqQwmPkRERKQymPgQERGRymDiQ0RERCqDiQ8RERGpDCY+REREpDKY+BAREZHKKJGzs1PxMnRhm8IOgYiIVAR7fIiIiEhlMPEhIiIilcHEh4iIiFQGEx8iIiJSGUx8iIiISGUw8SEiIiKVwcSHiIiIVAaf41MArs/vWdghEBERURbY40NEREQqg4kPERERqQwmPkRERKQymPgQERGRymDiQ0RERCqDiQ8RERGpDCY+REREpDKY+BAREZHKYOJDREREKoOJDxEREakMJj5ERESkMpj4EBERkcpg4kNEREQqg7OzExFlMnRhm8IOgYgKCHt8iIiISGUw8SEiIiKVwcSHiIiIVAYTHyIiIlIZTHyIiIhIZTDxISIiIpXBxIeIiIhUBhMfIiIiUhlMfIiIiEhlMPEhIiIilVGiEh8rKytIJBK515w5cwo7LCIiIioiStxcXdOmTUP//v3F9/r6+oUYDRERERUlJS7x0dfXh5mZWWGHQUREREVQibrUBQBz5sxB6dKlUbNmTcyfPx+pqak51k9OTkZ8fLzci4iIiEqmEtXjM3z4cNSqVQvGxsYICgrChAkT8Pz5cyxatCjbdWbPno2pU6d+xyiJqCB4/Hu2sEMgomKgyPf4jB8/XmHAcuZXWFgYAODXX3+Fp6cnqlWrhoEDB2LhwoVYvnw5kpOTs21/woQJiIuLE1+PHz/+XrtGRERE31mR7/EZNWoU/Pz8cqxjY2OTZXm9evWQmpqK6OhoODo6ZllHKpVCKpV+a5hERERUDBT5xMfExAQmJib5Wjc4OBhqamowNTVVclRERERUHBX5xCe3Ll68iMuXL6NRo0bQ19fHxYsXMXLkSHTv3h2lSpUq7PCIiIioCCgxiY9UKsW2bdvg7++P5ORkWFtbY+TIkfj1118LOzQiIiIqIkpM4lOrVi1cunSpsMMgIiKiIqzI39VFREREpCxMfIiIiEhlMPEhIiIilcHEh4iIiFQGEx8iIiJSGd+U+ERERODo0aP477//AACCICglKCIiIqKCkK/E582bN2jatCkcHBzQsmVLPH/+HADQt29fjBo1SqkBEhERESlLvhKfkSNHQkNDAzExMdDV1RXLf/rpJxw5ckRpwREREREpU74eYHjs2DEcPXoU5cuXlyu3t7fHo0ePlBIYERERkbLlq8cnMTFRrqcnw9u3bznTORERERVZ+Up83N3dsXHjRvG9RCJBeno65s2bh0aNGiktOCIiIiJlytelrnnz5qFJkya4du0aUlJSMHbsWNy9exdv377FhQsXlB0jERERkVLkq8enatWqePDgAdzc3NCuXTskJibixx9/xM2bN2Fra6vsGImIiIiUIt+zsxsaGmLixInKjIWIiIioQOWrxycgIAA7d+5UKN+5cyc2bNjwzUERERERFYR8JT6zZ89GmTJlFMpNTU0xa9asbw6KiIiIqCDkK/GJiYmBtbW1QnnFihURExPzzUERERERFYR8JT6mpqa4deuWQnlISAhKly79zUERERERFYR8JT5du3bF8OHDcfr0aaSlpSEtLQ2nTp3CL7/8gi5duig7RiIiIiKlyNddXdOnT0d0dDSaNGkCDY3PTaSnp6Nnz54c40NERERFVr4SHy0tLWzfvh3Tp09HSEgIdHR04OzsjIoVKyo7PiIiIiKlyfdzfADAwcEBDg4OyoqFiIiIqEDlK/FJS0tDYGAgTp48iZcvXyI9PV1u+alTp5QSHBEREZEy5Svx+eWXXxAYGIhWrVqhatWqkEgkyo6LiIiISOnylfhs27YNO3bsQMuWLZUdDxEREVGBydft7FpaWrCzs1N2LEREREQFKl+Jz6hRo7B06VIIgqDseIiIiIgKTL4udZ0/fx6nT5/G4cOHUaVKFWhqasot37Nnj1KCIyIiIlKmfCU+RkZGaN++vbJjISIiIipQ+Up8AgIClB0HERERUYHL1xgfAEhNTcWJEyewdu1afPjwAQDw7NkzJCQkKC04IiIiImXKV4/Po0eP4O3tjZiYGCQnJ6NZs2bQ19fH3LlzkZycjDVr1ig7TiIiIqJvlq8en19++QW1a9fGu3fvoKOjI5a3b98eJ0+eVFpwRERERMqUrx6fc+fOISgoCFpaWnLlVlZWePr0qVICIyIiIlK2fPX4pKenIy0tTaH8yZMn0NfX/+agiIiIiApCvhKf5s2bY8mSJeJ7iUSChIQETJkyhdNYEBERUZGVr0tdCxYsgLe3NypXroykpCR069YN4eHhKFOmDLZu3arsGImIiL6bC8MuKKWdszs9lNIOKVe+Eh9LS0uEhIRg+/btCAkJQUJCAvr27QtfX1+5wc5ERERERUmeE59Pnz6hUqVKOHDgAHx9feHr61sQcREREREpXZ7H+GhqaiIpKakgYiEiIiIqUPka3DxkyBDMnTsXqampyo6HiIiIqMDka4zP1atXcfLkSRw7dgzOzs7Q09OTW87Z2YmIiKgoyvfs7B06dFB2LEREREQFirOz50NaWho+ffpU2GHkmamOqdLa4jivgqOpqQl1dfXCDoOIqETKV+IDfJ6d/cyZM4iMjES3bt2gr6+PZ8+ewcDAADKZTJkxFhmCICA2Nhbv378v7FDyZbjzcKW1FRUVpbS2SJGRkRHMzMwgkUgKOxQiohKFs7PnQUbSY2pqCl1d3WL3pSS8FpTWlnUZa6W1Rf9HEAR8/PgRL1++BACYm5sXckRERCVLvhKfjNnZQ0JCULp0abG8ffv26N+/v9KC+9LMmTNx8OBBBAcHQ0tLK8tel5iYGAwaNAinT5+GTCZDr169MHv2bGho5LtjS5SWliYmPV/uc3Gippmvm/iypK2trbS2SF7GQ0BfvnwJU1NTXvYiIlKiYjM7e0pKCjp16oT69etj3bp1CsvT0tLQqlUrmJmZISgoCM+fP0fPnj2hqamJWbNmffP2M8b06OrqfnNbRF+TcZ59+vSJiQ8RkRIVm9nZp06dipEjR8LZ2TnL5ceOHcO9e/ewefNm1KhRAy1atMD06dOxcuVKpKSkKC2O4nZ5i4onnmdERAWjxMzOfvHiRTg7O6Ns2bJimZeXF+Lj43H37t1s10tOTkZ8fLzci4iIiEqmfCU+CxcuxIULF+RmZ8+4zDV37lxlx5grsbGxckkPAPF9bGxstuvNnj0bhoaG4svS0rJA46TPzpw5A4lEkqc75KysrOQSbiIiorzKV+JTvnx5hISEYOLEiRg5ciRq1qyJOXPm4ObNmzA1zf2zYsaPHw+JRJLjKywsLD8h5tqECRMQFxcnvh4/flyg2ysuJgyfACczJ/iP9VdYNm38NEgkEvj5+X33uIiIiL5Frgc316pVCydPnkSpUqUwbdo0jB49+ptnZx81atRXvzxtbGxy1ZaZmRmuXLkiV/bixQtxWXakUimkUmmutqFqzC3McWjfIYyfOh7aOp/v4kpOSsbBvQdRoUKFQo6OiIgo73Ld4xMaGorExEQAnwcaJyQkfPPGTUxMUKlSpRxfme8cy079+vVx+/Zt8fknAHD8+HEYGBigcuXK3xyrKqrsXBlm5cxw/NBxsez4oeMwtzBHzZo1xbLk5GQMHz4cpqam0NbWhpubG65evSrX1qFDh+Dg4AAdHR00atQI0dHRCts7f/483N3doaOjA0tLSwwfPlw854iIiJQh1z0+NWrUQO/eveHm5gZBELBgwYJsn9A8efJkpQWYISYmBm/fvkVMTAzS0tIQHBwMALCzs4NMJkPz5s1RuXJl9OjRA/PmzUNsbCx+//13DBkyhD063+DHrj9iz7Y9aNOhDQBg99bdaN+lPUKvh4p1xo4di927d2PDhg2oWLEi5s2bBy8vL0RERMDY2BiPHz/Gjz/+iCFDhmDAgAG4du0aRo0aJbedyMhIeHt7Y8aMGVi/fj1evXqFoUOHYujQoSo/RQoRESlPrhOfwMBATJkyBQcOHIBEIsHhw4ezfDCgRCIpkMRn8uTJ2LBhg/g+o8fh9OnT8PT0hLq6Og4cOIBBgwahfv360NPTQ69evTBt2jSlx6JK2nZoi8WzFuPp48/PZ7p59SYWrVkkJj6JiYlYvXo1AgMD0aJFCwDAn3/+iePHj2PdunUYM2YMVq9eDVtbWyxcuBAA4OjoiNu3b8sNhJ89ezZ8fX0xYsQIAIC9vT2WLVsGDw8PrF69mg9MJCIipch14uPo6Iht27YBANTU1HDy5Mk8DWT+VoGBgQgMDMyxTsWKFXHo0KHvE5CKMC5jDI+mHti3fR8EQYBHEw+UKl1KXB4ZGYlPnz7B1dVVLNPU1ETdunURGvo5OQoNDUW9evXk2q1fv77c+5CQENy6dQtbtmwRywRBQHp6OqKiouDk5FQQu0dERComX4Obp0yZUmInIiVFP3b5ETN+mwEAmDR7UoFsIyEhAT///DOGD1ecSJUDqYmISFnyNbh52rRpShncTMWDe2N3fPr0CampqXBr5Ca3zNbWFlpaWrhw4YJY9unTJ1y9elUcVO7k5KRwx92lS5fk3teqVQv37t2DnZ2dwiu3A9yJiIi+ptgMbqbCo66ujoPnDoo/f0lPTw+DBg3CmDFjYGxsjAoVKmDevHn4+PEj+vbtCwAYOHAgFi5ciDFjxqBfv364fv26wmXLcePG4YcffsDQoUPRr18/6Onp4d69ezh+/DhWrFjxXfaTiIhKvmIzuJkKl0w/+0ubc+bMQXp6Onr06IEPHz6gdu3aOHr0KEqV+jwWqEKFCti9ezdGjhyJ5cuXo27dupg1axb69OkjtlGtWjWcPXsWEydOhLu7OwRBgK2tLX766acC3zciIlIdEkEQhLyupKamhtjY2O86uPl7iY+Ph6GhIeLi4mBgYCCWJyUlISoqCtbW1sX2DqOwF8p7CnalspWU1hYpUvb55jJmoxKi+uz6/J5Ka4uoJDvb0ENpbXn8e1ZpbZVU2X1/Z5brHp8vpaen5zswIiIiosKS68Rn//79aNGiBTQ1NbF///4c67Zt2/abAyMiIiJStlwnPj4+PuLlLR8fn2zrSSQSpKWlKSM2IiIiIqXKdeLz5eUtXuoiIiKi4ijPY3zS09MRGBiIPXv2IDo6GhKJBDY2NujQoQN69OgBiURSEHESERERfbNcP8AQ+DyFQNu2bdGvXz88ffoUzs7OqFKlCqKjo+Hn54f27dsXVJxERERE3yxPPT6BgYH4999/cfLkSTRq1Ehu2alTp+Dj44ONGzeiZ0/e7kpERERFT556fLZu3YrffvtNIekBgMaNG2P8+PFyk0wSERERFSV5Snxu3boFb2/vbJe3aNECISEh3xwUERERUUHIU+Lz9u1blC1bNtvlZcuWxbt37745KCIiIqKCkKcxPmlpaVnOz5VBXV0dqamp3xxUcaPM6QByIz9TBsTGxmLWpFk4e+IsYp/HQl9fHxWsK6BNhzbw6ewDHV2dAoj0+/Pz88OGDRsUyitXroy7d+9mWcfY2Bh16tTBvHnzUK1ate8WKxERfX95SnwEQYCfnx+kUmmWy5OTk5USFCnXw4cP4erqCl19XYyYMAIOTg7QkmrhQegD7Ni0A2XNy6KxV+PCDlMpli5dijlz5ojvU1NTUb16dXTq1Emunre3NwICAgB8Tgp///13tG7dGjExMd81XiIi+r7ydKmrV69eMDU1haGhYZYvU1NT3tFVBA0ePBgaGhrYeWQnWrRrAVsHW1hWtEQT7yZYu2UtGjX/v8Hq8XHx+P3X39GgcgPUtqsNvw5+CLv7f5Obrpi/Au2btMf69etRoUIFyGQyDB48GGlpaZg3bx7MzMxgamqKmTNnysUgkUiwdu1atG7dGrq6unBycsLFixcREREBT09P6OnpoUGDBoiMjBTXiYyMRLt27VC2bFnIZDLUqVMHJ06cyHFfDQ0NYWZmJr6uXbuGd+/eoXfv3nL1pFKpWKdGjRoYP348Hj9+jFevXn3LoSYioiIuTz0+Gf8hU/Hx5s0bHDt2DLNmzYKunm6Wdb586OSI/iOgra2NP/7+AzIDGXZs3IHenXrj8IXDMCplBACIiY7B4cOHceTIEURGRqJjx454+PAhHBwccPbsWQQFBaFPnz5o2rQp6tWrJ7Y9ffp0LFq0CIsWLcK4cePQrVs32NjYYMKECahQoQL69OmDoUOH4vDhwwCAhIQEtGzZEjNnzoRUKsXGjRvRpk0b3L9/HxUqVMjV/q9btw5NmzZFxYoVs62TkJCAzZs3w87ODqVLl85Vu0REVDzla3Z2Kj4iIiIgCAIcHR3lyutXro+UpBQAQNfeXTF60mhcv3wdt2/exoU7F6Al1QIAjPUfi5NHTuLYgWPo3KMzAEBIF7B+/Xro6+ujcuXKaNSoEe7fv49Dhw5BTU0Njo6OmDt3Lk6fPi2X+PTu3RudO39uY9y4cahfvz4mTZoELy8vAMAvv/wi1zNTvXp1VK9eXXw/ffp07N27F/v378fQoUO/uu/Pnj3D4cOH8ffffyssO3DgAGQyGQAgMTER5ubmOHDgANTU8tQJSkRExQwTHxW14/AOpKenY+zgsUhJ+ZwAhd0Nw8fEj6jvVF+ublJSEmKi/2/sSznLctDX1xffly1bFurq6nJJQ9myZfHy5Uu5dr4cOJxxd6Czs7NcWVJSEuLj42FgYICEhAT4+/vj4MGDeP78OVJTU/Hff//lehzOhg0bYGRklOWkuo0aNcLq1asBAO/evcOqVavQokULXLlyJcfeISIiKt6Y+JRwdnZ2kEgkuH//PpwaOInllhUtAQBS7f8bqP4x8SNMyppgwx7Fu6IMDAzEnzU1NeWWSSSSLMsyT2b7ZZ2My2tZlWWsN3r0aBw/fhwLFiyAnZ0ddHR00LFjRzFRy4kgfO6V6tGjB7S0tBSW6+npwc7OTnz/119/wdDQEH/++SdmzJjx1faJiKh4YuJTwpUuXRrNmjXDihUr0Lxz82zH+QBA5WqV8frla2ioa8CigsV3jDJrFy5ckJsDLiEhAdHR0bla9+zZs4iIiEDfvn1zVV8ikUBNTQ3//fdffsMlIqJigAMaVMCqVauQmpqKTt6dcGjfIUQ+iERURBT279qPqIgoqKupAwAaNGyAGrVrYGjvobhw5gKexjzFzas3sWT2EtwJvvPd47a3t8eePXsQHByMkJAQdOvWTaEXKTvr1q1DvXr1ULVq1SyXJycnIzY2FrGxsQgNDcWwYcOQkJCANm3aKHMXiIioiGGPjwqwtbXFzZs3Meb3MVg8azFePH8BTS1N2DnYofeg3ujq1xXA/7/lfMtaLJm9BL+N+A3v3rxDGdMyqP1DbZQ2+f53Oy1atAh9+vRBgwYNUKZMGYwbNw7x8fFfXS8uLg67d+/G0qVLs61z5MgRmJubAwD09fVRqVIl7Ny5E56ensoKn4iIiiCJIAhCYQdRlMTHx8PQ0BBxcXFy41qSkpIQFRUFa2traGtrF2KE+Rf2IuzrlXKpUtlKSmuLFCn7fFPm08Xz8+RwIlV0tqGH0try+Pes0toqqbL7/s6Ml7qIiIhIZTDxISIiIpXBxIeIiIhUBhMfIiIiUhlMfIiIiEhlMPEhIiIilcHEh4iIiFQGEx8iIiJSGUx8iIiISGUw8SEFPdv3xKxJswpt+56enhgxYkShbZ+IiEouztWlBDHTnL/r9ipMvp2n+n5+ftiwYQN+6vkT/Of5yy2bNn4atgZuhU9nH8xeNhsAsGz9Mmhqauap/ffv32Pfvn15iutbbNmyBfPmzUN4eDgMDQ3RokULzJ8/H6VL525OMX9/f0ydOlWhXFdXF4mJiVnWMTAwQLVq1TBjxgx4eCjvUfRERPT9MPFREZaWljjyzxEErAmAjo4OgM/zQR3edxgVKlSAoY7h/82/VbZgYvj06VOeEqrsXLhwAT179sTixYvRpk0bPH36FAMHDkT//v2xZ8+eXLUxevRoDBw4UK6sSZMmqFOnjlxZlSpVcOLECQDA27dvsWDBArRu3RpPnjyBoaHhN+8LERF9X7zUpSJq1aoFS0tLucRgz549qFChAmrWrClX98tLTWFhYdDV1cXff/8tLt+xYwd0dHRw7949+Pv7Y8OGDfjnn38gkUggkUhw5swZREdHQyKRYPv27fDw8IC2tja2bNmCN2/eoGvXrrCwsICuri6cnZ2xdevWPO3LxYsXYWVlheHDh8Pa2hpubm74+eefceXKFQCfE7oqVapgwIAB4jqRkZHQ19fH+vXrAQAymQxmZmbi68WLF7h37x769u0rty0NDQ2xTuXKlTFt2jQkJCTgwYMHeYqZiIiKBiY+KqRPnz4ICAgQ369fvx69e/fOcZ1KlSphwYIFGDx4MGJiYvDkyRMMHDgQc+fOReXKlTF69Gh07twZ3t7eeP78OZ4/f44GDRqI648fPx6//PILQkND4eXlhaSkJLi4uODgwYO4c+cOBgwYgB49eohJS27Ur18fjx8/xqFDhyAIAl68eIFdu3ahZcuWACAmWRkJWVpaGrp3745mzZqhT58+Wbb5119/wcHBAe7u7tluNzk5GQEBATAyMoKjo2Ou4yUioqKDl7pUSPfu3TFhwgQ8evQIwOdLRtu2bcOZM2dyXG/w4ME4dOgQunfvDi0tLdSpUwfDhg0D8LnnREdHB8nJyTAzM1NYd8SIEfjxxx/lykaPHi3+PGzYMBw9ehQ7duxA3bp1c7Ufrq6u2LJlC3766SckJSUhNTUVbdq0wcqVK8U6NWrUwIwZM9CvXz906dIFjx49woEDB7JsLykpCVu2bMH48eMVlt2+fRsymQwA8PHjR+jr62P79u0wMDDIVaxERFS0MPFRISYmJmjVqhUCAwMhCAJatWqFMmXK5Grd9evXw8HBAWpqarh79y4kEkmu1qtdu7bc+7S0NMyaNQs7duzA06dPkZKSguTkZOjq6uZ6P+7du4dffvkFkydPhpeXF54/f44xY8Zg4MCBWLdunVhv1KhR2LdvH1asWIHDhw9nO/B57969+PDhA3r16qWwzNHREfv37wcAfPjwAdu3b0enTp1w+vRphX0jIqKij4mPiunTpw+GDh0KAHI9JF8TEhKCxMREqKmp4fnz5zA3N8/Venp6enLv58+fj6VLl2LJkiVwdnaGnp4eRowYgZSUlFzHMnv2bLi6umLMmDEAgGrVqkFPTw/u7u6YMWOGGNvLly/x4MEDqKurIzw8HN7e3lm299dff6F169YoW1ZxVLeWlhbs7OzE9zVr1sS+ffuwZMkSbN68OdcxExFR0cDER8V4e3sjJSUFEokEXl5euVrn7du38PPzw8SJE/H8+XP4+vrixo0b4t1hWlpaSEtLy1VbFy5cQLt27dC9e3cAQHp6Oh48eIDKlSvneh8+fvwIDQ35U1ddXR0AIAiCWNanTx84Ozujb9++6N+/P5o2bQonJye59aKionD69GmxVyc31NXV8d9//+W6PhERFR0c3Kxi1NXVERoainv37onJwtcMHDgQlpaW+P3337Fo0SKkpaXJjdOxsrLCrVu3cP/+fbx+/RqfPn3Kti17e3scP34cQUFBCA0Nxc8//4wXL17kaR/atGmDPXv2YPXq1Xj48CEuXLiA4cOHo27duihXrhyAz71ZFy9exIYNG+Dr6wsfHx/4+voq9CytX78e5ubmaNGiRZbbSk1NRWxsLGJjYxEeHo4ZM2bg3r17aNeuXZ5iJiKiooGJjwoyMDDI9eDcjRs34tChQ9i0aRM0NDSgp6eHzZs3488//8Thw4cBAP3794ejoyNq164NExMTXLhwIdv2fv/9d9SqVQteXl7w9PSEmZkZfHx88hS/n58fFi1ahBUrVqBq1aro1KkTHB0dxVv1w8LCMGbMGKxatQqWlpYAgFWrVuH169eYNGmS2E56ejoCAwPh5+eXbRJ49+5dmJubw9zcHDVq1MCOHTuwevVq9OzZM08xExFR0SARvrw2UITNnDkTBw8eRHBwMLS0tPD+/XuFOlkNuN26dSu6dOmS6+3Ex8fD0NAQcXFxcslBUlISoqKiYG1tDW1t7XztA1FuKft8cxmzUQlRfXZ9PpM+otw421B5T3j3+Pes0toqqbL7/s6s2IzxSUlJQadOnVC/fn25O3cyCwgIkBvEamRk9B2iIyIiouKg2CQ+GXMmBQYG5ljPyMgoy+fJEBEREZW4MT5DhgxBmTJlULduXaxfvx7F5EoeERERfQfFpscnN6ZNm4bGjRtDV1cXx44dw+DBg5GQkIDhw4dnu05ycjKSk5PF9/Hx8d8jVCIiIioEhdrjM378eHFiy+xeYWFhuW5v0qRJcHV1Rc2aNTFu3DiMHTsW8+fPz3Gd2bNnw9DQUHxl3AVEREREJU+h9viMGjUKfn5+OdaxsbHJd/v16tXD9OnTkZycDKlUmmWdCRMm4NdffxXfx8fHM/khIiIqoQo18TExMYGJiUmBtR8cHIxSpUplm/QAgFQqzXE5ERERlRzFZoxPTEwM3r59i5iYGKSlpSE4OBgAYGdnB5lMhv/973948eIFfvjhB2hra+P48eOYNWuW3BOGiYiISLUVm7u6Jk+ejJo1a2LKlClISEhAzZo1UbNmTVy7dg0AoKmpiZUrV6J+/fqoUaMG1q5di0WLFmHKlCmFHHnxIZFIsG/fvgLfjqenJ0aMGFHg2ylIVlZWWLJkifj+ex07IiL6NsUm8QkMDIQgCAovT09PAJ8n37x58yY+fPiAhIQEBAcH4+eff4aaWrHZxQIVGxuLYcOGwcbGBlKpFJaWlmjTpg1OnjxZ2KHly88//wxbW1vo6OjAxMQE7dq1++pA+OwSrsDAwDw/6PLq1asYMGBAntYhIqLCV2wudRVlrstdv+v2LgzLfi6srERHR8PV1RVGRkaYP38+nJ2d8enTJxw9ehRDhgzJ051zRYWLiwt8fX1RoUIFvH37Fv7+/mjevDmioqJyPfnqtyjIsWlERFRw2B2iAgYPHgyJRIIrV66gQ4cOcHBwQJUqVfDrr7/i0qVL2a43btw4ODg4QFdXFzY2Npg0aZLczOt+fn4KE4yOGDFC7IUDgMTERPTs2RMymQzm5uZYuHChwnaSk5MxevRoWFhYQE9PD/Xq1cOZM2dy3KcBAwagYcOGsLKyQq1atTBjxgw8fvwY0dHRuTkkOYqMjES7du1QtmxZyGQy1KlTBydOnJCrk/lSV2ZTpkyBubk5bt26lWWP0r59+7KcW46IiAoWE58S7u3btzhy5AiGDBkCPT09heU5XeLR19dHYGAg7t27h6VLl+LPP//E4sWL87T9MWPG4OzZs/jnn39w7NgxnDlzBjdu3JCrM3ToUFy8eBHbtm3DrVu30KlTJ3h7eyM8PDxX20hMTERAQACsra2V8iiChIQEtGzZEidPnsTNmzfh7e2NNm3aICYm5qvrCoKAYcOGYePGjTh37hyqVav2zfEQEZHy8FJXCRcREQFBEFCpUqU8r/v777+LP1tZWWH06NHYtm0bxo4dm6v1ExISsG7dOmzevBlNmjQBAGzYsAHly5cX68TExCAgIAAxMTEoV64cAGD06NE4cuQIAgICMGvWrGzbX7VqFcaOHYvExEQ4Ojri+PHj0NLSyjGmVatW4a+//pIrS01NlZsBvXr16qhevbr4fvr06di7dy/279+PoUOHZtt2amoqunfvjps3b+L8+fOwsLDIMRYiIvr+mPiUcN8yV9n27duxbNkyREZGIiEhAampqTAwMMj1+pGRkUhJSUG9evXEMmNjYzg6Oorvb9++jbS0NDg4OMitm5ycjNKlS+fYvq+vL5o1a4bnz59jwYIF6Ny5My5cuCCXxGS1zsSJE+XK9uzZI5dgJSQkwN/fHwcPHsTz58+RmpqK//7776s9PiNHjoRUKsWlS5dQpkyZHOsSEVHhYOJTwtnb2+d56g8AuHjxInx9fTF16lR4eXnB0NAQ27Ztkxujo6amppBYfTkGKDcSEhKgrq6O69evKwxKlslkOa6bMc2Ivb09fvjhB5QqVQp79+5F165dc1zHzs5OrszU1FTu/ejRo3H8+HEsWLAAdnZ20NHRQceOHZGSkpJjPM2aNcPWrVtx9OhR+Pr6iuXKOE5ERKQcTHxKOGNjY3h5eWHlypUYPny4wjif9+/fZznOJygoCBUrVpTrHXn06JFcHRMTE9y5c0euLDg4GJqamgAAW1tbaGpq4vLly6hQoQIA4N27d3jw4AE8PDwAADVr1kRaWhpevnwJd3f3fO9nxuMNvpxwNr8uXLgAPz8/tG/fHsDn5Cw3g6bbtm2LNm3aoFu3blBXV0eXLl0AfD5OHz58QGJionj8Mx7ASURE3xcHN6uAlStXIi0tDXXr1sXu3bsRHh6O0NBQLFu2DPXr189yHXt7e8TExGDbtm2IjIzEsmXLsHfvXrk6jRs3xrVr17Bx40aEh4djypQpcomQTCZD3759MWbMGJw6dQp37tyBn5+f3LOVHBwc4Ovri549e2LPnj2IiorClStXMHv2bBw8eDDL2B4+fIjZs2fj+vXriImJQVBQEDp16gQdHR20bNnym4+Xvb099uzZg+DgYISEhKBbt25IT0/P1brt27fHpk2b0Lt3b+zatQvA5znjdHV18dtvvyEyMhJ///03AgMDvzlOIiLKOyY+KsDGxgY3btxAo0aNMGrUKFStWhXNmjXDyZMnsXr16izXadu2LUaOHImhQ4eiRo0aCAoKwqRJk+TqeHl5YdKkSRg7dizq1KmDDx8+oGfPnnJ15s+fD3d3d7Rp0wZNmzaFm5sbXFxc5OoEBASgZ8+eGDVqFBwdHeHj44OrV6+KvUSZaWtr49y5c2jZsiXs7Ozw008/QV9fH0FBQQqXrfJj0aJFKFWqFBo0aIA2bdrAy8sLtWrVyvX6HTt2xIYNG9CjRw/s2bMHxsbG2Lx5Mw4dOgRnZ2ds3boV/v7+3xwnERHlnUT4ltGvJVB8fDwMDQ0RFxcnN5A3KSkJUVFRsLa2znHwLJEyKPt8cxmzUQlRfXZ9fs+vVyIinG3oobS2PP49q7S2Sqrsvr8zY48PERERqQwmPkRERKQymPgQERGRymDiQ0RERCqDiQ8RERGpDCY+REREpDKY+BAREZHKYOJDREREKoOJDxEREakMJj4EiUSCffv2FXYYREREBY6Jjwrw8/ODj49PYYeRLz///DNsbW2ho6MDExMTtGvXDmFhYYUdFhERFVMahR1ASaDM+VhyQ5XmbHFxcYGvry8qVKiAt2/fwt/fH82bN0dUVBTU1dULOzwiIipm2ONDCsaNGwcHBwfo6urCxsYGkyZNwqdPn8Tl/v7+qFGjBtavX48KFSpAJpNh8ODBSEtLw7x582BmZgZTU1PMnDlTrt1FixbB2dkZenp6sLS0xODBg5GQkJBjLAMGDEDDhg1hZWWFWrVqYcaMGXj8+DGio6MLYteJiKiEY48PKdDX10dgYCDKlSuH27dvo3///tDX18fYsWPFOpGRkTh8+DCOHDmCyMhIdOzYEQ8fPoSDgwPOnj2LoKAg9OnTB02bNkW9evUAAGpqali2bBmsra3x8OFDDB48GGPHjsWqVatyFVdiYiICAgJgbW0NS0vLAtl3IiIq2Zj4kILff/9d/NnKygqjR4/Gtm3b5BKf9PR0rF+/Hvr6+qhcuTIaNWqE+/fv49ChQ1BTU4OjoyPmzp2L06dPi4nPiBEj5NqdMWMGBg4c+NXEZ9WqVRg7diwSExPh6OiI48ePQ0tLS7k7TUREKoGJDynYvn07li1bhsjISCQkJCA1NRUGBgZydaysrKCvry++L1u2LNTV1aGmpiZX9vLlS/H9iRMnMHv2bISFhSE+Ph6pqalISkrCx48foaurm208vr6+aNasGZ4/f44FCxagc+fOuHDhArS1tZW410REpAo4xofkXLx4Eb6+vmjZsiUOHDiAmzdvYuLEiUhJSZGrp6mpKfdeIpFkWZaeng4AiI6ORuvWrVGtWjXs3r0b169fx8qVKwFAoe3MDA0NYW9vj4YNG2LXrl0ICwvD3r17v3VXiYhIBbHHh+QEBQWhYsWKmDhxolj26NGjb273+vXrSE9Px8KFC8VeoR07duS5HUEQIAgCkpOTvzkmIiJSPUx8VERcXByCg4PlykqXLq0wSNje3h4xMTHYtm0b6tSpg4MHDyqld8XOzg6fPn3C8uXL0aZNG1y4cAFr1qzJcZ2HDx9i+/btaN68OUxMTPDkyRPMmTMHOjo6aNmy5TfHREREqoeXulTEmTNnULNmTbnX1KlTFeq1bdsWI0eOxNChQ1GjRg0EBQVh0qRJ37z96tWrY9GiRZg7dy6qVq2KLVu2YPbs2Tmuo62tjXPnzqFly5aws7PDTz/9BH19fQQFBcHU1PSbYyIiItUjEQRBKOwgipL4+HgYGhoiLi5ObkBvUlISoqKiYG1tzUG1VOCUfb65jNmohKg+uz6/p9LaIirJlPlwW1V6cG1+Zff9nRl7fIiIiEhlMPEhIiIilcHEh4iIiFQGEx8iIiJSGUx8iIiISGUw8SEiIiKVwcSHiIiIVAYTHyIiIlIZTHyIiIhIZTDxISIiIpXBxEeFxMbG4pdffoGdnR20tbVRtmxZuLq6YvXq1fj48SMAwMrKChKJBBKJBHp6eqhVqxZ27typsCyrl5+fn8I2z58/D1dXV5QuXRo6OjqoVKkSFi9enGOc0dHRWbZ/6dIlpR8TIiJSLZydXQlWjPrfd93e0IVt8rzOw4cP4erqCiMjI8yaNQvOzs6QSqW4ffs2/vjjD1hYWKBt27YAgGnTpqF///6Ij4/HwoUL8dNPP8HCwgJXr15FWloaACAoKAgdOnTA/fv3xTlRdHR0FLarp6eHoUOHolq1atDT08P58+fx888/Q09PDwMGDMgx5hMnTqBKlSri+9KlS+d5v4mIiL7ExEdFDB48GBoaGrh27Rr09PTEchsbG7Rr1w5fzlWrr68PMzMzmJmZYeXKldi8eTP+97//yc2mbmxsDAAwNTWFkZFRttvNmAk+g5WVFfbs2YNz5859NfEpXbo0zMzM8rqrRERE2eKlLhXw5s0bHDt2DEOGDJFLer4kkUiyLNfQ0ICmpiZSUlKUEsvNmzcRFBQED4+vz1rctm1bmJqaws3NDfv371fK9omISLUx8VEBEREREAQBjo6OcuVlypSBTCaDTCbDuHHjFNZLSUnB7NmzERcXh8aNG39TDOXLl4dUKkXt2rUxZMgQ9OvXL9u6MpkMCxcuxM6dO3Hw4EG4ubnBx8eHyQ8REX2zYpH4REdHo2/fvrC2toaOjg5sbW0xZcoUhV6IW7duwd3dHdra2rC0tMS8efMKKeLi4cqVKwgODkaVKlWQnJwslo8bNw4ymQy6urqYO3cu5syZg1atWn21vYwkSiaTYeDAgXLLzp07h2vXrmHNmjVYsmQJtm7dmm07ZcqUwa+//op69eqhTp06mDNnDrp374758+fnf2eJiIhQTMb4hIWFIT09HWvXroWdnR3u3LmD/v37IzExEQsWLAAAxMfHo3nz5mjatCnWrFmD27dvo0+fPjAyMvrqWJKSzs7ODhKJBPfv35crt7GxAaA4KHnMmDHw8/ODTCZD2bJls70MlllwcLD4c8aA5wzW1tYAAGdnZ7x48QL+/v7o2rVrrvehXr16OH78eK7rExERZaVYJD7e3t7w9vYW39vY2OD+/ftYvXq1mPhs2bIFKSkpWL9+PbS0tFClShUEBwdj0aJFKp/4lC5dGs2aNcOKFSswbNiwbMf5ZChTpgzs7OzyvJ3crpOeni7Xw5QbwcHBMDc3z3NMREREXyoWiU9W4uLixDuLAODixYto2LAhtLS0xDIvLy/MnTsX7969Q6lSpQojzCJj1apVcHV1Re3ateHv749q1apBTU0NV69eRVhYGFxcXApkuytXrkSFChVQqVIlAMC///6LBQsWYPjw4WKdFStWYO/evTh58iQAYMOGDdDS0hLvBtuzZw/Wr1+Pv/76q0BiJCIi1VEsE5+IiAgsX75c7O0BPj+cL+NySoayZcuKy7JLfJKTk+V6H+Lj4wsg4sJna2uLmzdvYtasWZgwYQKePHkCqVSKypUrY/To0Rg8eHCBbDc9PR0TJkxAVFQUNDQ0YGtri7lz5+Lnn38W67x+/RqRkZFy602fPh2PHj2ChoYGKlWqhO3bt6Njx44FEiMREakOifDlA1y+s/Hjx2Pu3Lk51gkNDRV7CwDg6dOn8PDwgKenp1wPQPPmzWFtbY21a9eKZffu3UOVKlVw7949ODk5Zdm+v78/pk6dqlAeFxcnN04lKSkJUVFRsLa2hra2dq73kSg/lH2+uYzZqISoPrs+v6fS2iIqyc42/PpjO3LL49+zSmurpIqPj4ehoaHC93dmhdrjM2rUqCynOfhSxgBcAHj27BkaNWqEBg0a4I8//pCrZ2ZmhhcvXsiVZbzP6SF4EyZMwK+//iq+j4+Ph6WlZW53gYiIiIqRQk18TExMYGJikqu6T58+RaNGjeDi4oKAgACoqcnfiV+/fn1MnDgRnz59gqamJgDg+PHjcHR0zHF8j1QqhVQqzf9OEBERUbFRLJ7j8/TpU3h6eqJChQpYsGABXr16hdjYWMTGxop1unXrBi0tLfTt2xd3797F9u3bsXTpUrneHCIiIlJtxWJw8/HjxxEREYGIiAiUL19eblnGECVDQ0NxWgYXFxeUKVMGkydPVvlb2YmIiOj/FIvEx8/P76tjgQCgWrVqOHfuXMEHRERERMVSsbjURURERKQMTHyIiIhIZTDxISIiIpXBxIeIiIhUBhMfIiIiUhlMfFRIbGwsfvnlF9jZ2UFbWxtly5aFq6srVq9ejY8fPwIArKysIJFIIJFIoKenh1q1amHnzp0Ky7J6fe3OuwsXLkBDQwM1atQo4D0lIiLKWrG4nb2om9n9+06eOXHzrjyv8/DhQ7i6usLIyAizZs2Cs7MzpFIpbt++jT/++AMWFhZo27YtAGDatGno378/4uPjsXDhQvz000+wsLDA1atXkZaWBgAICgpChw4dcP/+fXFOFB0dnWy3//79e/Ts2RNNmjRRmFqEiIjoe2HioyIGDx4MDQ0NXLt2DXp6emK5jY0N2rVrhy/nqtXX14eZmRnMzMywcuVKbN68Gf/73/8we/ZssY6xsTEAwNTUFEZGRl/d/sCBA9GtWzeoq6tj3759StsvIiKivOClLhXw5s0b8anWXyY9X5JIJFmWa2hoQFNTEykpKfnefkBAAB4+fIgpU6bkuw0iIiJlYOKjAiIiIiAIAhwdHeXKy5QpA5lMBplMhnHjximsl5KSgtmzZyMuLg6NGzfO17bDw8Mxfvx4bN68GRoa7GAkIqLCxcRHhV25cgXBwcGoUqUKkpOTxfJx48ZBJpNBV1cXc+fOxZw5c9CqVauvtpeRRMlkMgwcOBBpaWno1q0bpk6dCgcHh4LcFSIiolzhv+AqwM7ODhKJBPfv35crt7GxAaA4KHnMmDHw8/ODTCZD2bJls70MlllwcLD4s4GBAT58+IBr167h5s2bGDp0KAAgPT0dgiBAQ0MDx44dy3dPEhERUX4w8VEBpUuXRrNmzbBixQoMGzYs23E+GcqUKQM7O7s8byfzOunp6bh9+7Zc2apVq3Dq1Cns2rUL1tbWed4GERHRt2DioyJWrVoFV1dX1K5dG/7+/qhWrRrU1NRw9epVhIWFwcXFRenbVFNTQ9WqVeXKTE1Noa2trVBORET0PTDxURG2tra4efMmZs2ahQkTJuDJkyeQSqWoXLkyRo8ejcGDBxd2iERERAVOInz5ABdCfHw8DA0NERcXJz6YDwCSkpIQFRUFa2traGtrF2KEpAqUfb65jNmohKg+uz6/p9LaIirJzjb0UFpbHv+eVVpbJVV239+Z8a4uIiIiUhlMfIiIiEhlMPEhIiIilcHEh4iIiFQGEx8iIiJSGUx8iIiISGUw8SEiIiKVwcSHiIiIVAYTHyIiIlIZTHxIJJFIsG/fvgLfjqenJ0aMGFHg2yEiIsqMc3WpiNjYWMycORMHDx7E06dPYWpqiho1amDEiBFo0qRJYYeXZz///DNOnDiBZ8+eQSaToUGDBpg7dy4qVaqU43p3797F1KlTcfr0acTHx6NixYro0qULxo8fD11d3e8U/ffHaSaIiD5j4qMEoTNPfdftOU1snKf60dHRcHV1hZGREebPnw9nZ2d8+vQJR48exZAhQxAWFlZAkRYcFxcX+Pr6okKFCnj79i38/f3RvHlzREVFQV1dPct1Ll26hKZNm6Jp06Y4ePAgypYtiytXrmDUqFE4efIkTp8+DS0tre+8J0RE9D3xUpcKGDx4MCQSCa5cuYIOHTrAwcEBVapUwa+//opLly5lu964cePg4OAAXV1d2NjYYNKkSfj06ZO43M/PDz4+PnLrjBgxAp6enuL7xMRE9OzZEzKZDObm5li4cKHCdpKTkzF69GhYWFhAT08P9erVw5kzZ3LcpwEDBqBhw4awsrJCrVq1MGPGDDx+/BjR0dFZ1hcEAX379oWTkxP27NmDunXromLFiujUqRP+97//4eLFi1i8eLFYXyKRYPXq1WjRogV0dHRgY2ODXbt2icvPnDkDiUSC9+/fi2XBwcGQSCRiDI8ePUKbNm1QqlQp6OnpoUqVKjh06FCO+0VERAWLiU8J9/btWxw5cgRDhgyBnp6ewnIjI6Ns19XX10dgYCDu3buHpUuX4s8//5RLDnJjzJgxOHv2LP755x8cO3YMZ86cwY0bN+TqDB06FBcvXsS2bdtw69YtdOrUCd7e3ggPD8/VNhITExEQEABra2tYWlpmWSc4OBj37t3Dr7/+CjU1+dO+evXqaNq0KbZu3SpXPmnSJHTo0AEhISHw9fVFly5dEBoamut9HzJkCJKTk/Hvv//i9u3bmDt3LmQyWa7XJyIi5WPiU8JFRERAEISvjn3Jyu+//44GDRrAysoKbdq0wejRo7Fjx45cr5+QkIB169ZhwYIFaNKkCZydnbFhwwakpqaKdWJiYhAQEICdO3fC3d0dtra2GD16NNzc3BAQEJBj+6tWrYJMJoNMJsPhw4dx/PjxbC9VPXjwAADg5OSU5XInJyexToZOnTqhX79+cHBwwPTp01G7dm0sX7481/sfExMDV1dXODs7w8bGBq1bt0bDhg1zvT4RESkfx/iUcIIg5Hvd7du3Y9myZYiMjERCQgJSU1NhYGCQ6/UjIyORkpKCevXqiWXGxsZwdHQU39++fRtpaWlwcHCQWzc5ORmlS5fOsX1fX180a9YMz58/x4IFC9C5c2dcuHAB2tra2a6Tl+NRv359hffBwcG5Xn/48OEYNGgQjh07hqZNm6JDhw6oVq1artcnIiLlY49PCWdvbw+JRJLnAcwXL16Er68vWrZsiQMHDuDmzZuYOHEiUlJSxDpqamoKicSXY4ByIyEhAerq6rh+/TqCg4PFV2hoKJYuXZrjuoaGhrC3t0fDhg2xa9cuhIWFYe/evVnWzUissrtUFRoaqpB85STjctmX+5953/v164eHDx+iR48euH37dp57jIiISPmY+JRwxsbG8PLywsqVK5GYmKiw/MvBuV8KCgpCxYoVMXHiRNSuXRv29vZ49OiRXB0TExM8f/5cruzLHhFbW1toamri8uXLYtm7d+/kLinVrFkTaWlpePnyJezs7OReZmZmud5PQRAgCAKSk5OzXF6jRg1UqlQJixcvRnp6utyykJAQnDhxAl27dpUrzzzw+9KlS+KlMhMTEwCQ2/+seoMsLS0xcOBA7NmzB6NGjcKff/6Z630iIiLlY+KjAlauXIm0tDTUrVsXu3fvRnh4OEJDQ7Fs2TKFyzkZ7O3tERMTg23btiEyMhLLli1T6E1p3Lgxrl27ho0bNyI8PBxTpkzBnTt3xOUymQx9+/bFmDFjcOrUKdy5cwd+fn5yg4sdHBzg6+uLnj17Ys+ePYiKisKVK1cwe/ZsHDx4MMvYHj58iNmzZ+P69euIiYlBUFAQOnXqBB0dHbRs2TLLdSQSCdatW4d79+6hQ4cOuHLlCmJiYrBz5060adMG9evXV3io4s6dO7F+/Xo8ePAAU6ZMwZUrVzB06FAAgJ2dHSwtLeHv74/w8HAcPHhQ4Y61ESNG4OjRo4iKisKNGzdw+vTpbMcYERHR98ExPirAxsYGN27cwMyZMzFq1Cg8f/4cJiYmcHFxwerVq7Ncp23bthg5ciSGDh2K5ORktGrVCpMmTYK/v79Yx8vLC5MmTcLYsWORlJSEPn36oGfPnrh9+7ZYZ/78+UhISECbNm2gr6+PUaNGIS4uTm5bAQEBmDFjBkaNGoWnT5+iTJky+OGHH9C6dessY9PW1sa5c+ewZMkSvHv3DmXLlkXDhg0RFBQEU1PTbI9DgwYNcOnSJUydOhUtWrTAhw8fUKFCBfTq1QsTJkyAVCqVqz916lRs27YNgwcPhrm5ObZu3YrKlSsDADQ1NbF161YMGjQI1apVQ506dTBjxgx06tRJXD8tLQ1DhgzBkydPYGBgAG9v7zzfFUdExZfHv2cLOwTKgkT4ltGvJVB8fDwMDQ0RFxcnN5A3KSkJUVFRsLa2znHwLJUMEokEe/fuVXhO0ffC842IKG+y+/7OjJe6iIiISGUw8SEiIiKVwTE+RFngFWAiopKJPT5ERESkMpj4EBERkcpg4pNHvARC3wPPMyKigsHEJ5c0NTUBAB8/fizkSEgVZJxnGecdEREpBwc355K6ujqMjIzw8uVLAICuri4kEkkhR0UljSAI+PjxI16+fAkjIyOoq6sXdkhERCUKE588yJg7KiP5ISooRkZGeZqrjIiIcqdYJD7R0dGYPn06Tp06hdjYWJQrVw7du3fHxIkToaWlJdaxtrZWWPfixYv44YcflBKHRCKBubk5TE1N8zwLOVFuaWpqsqeHiKiAFIvEJywsDOnp6Vi7di3s7Oxw584d9O/fH4mJiViwYIFc3RMnTqBKlSri+9KlSys9HnV1dX4xERERFUPFIvHx9vaGt7e3+N7Gxgb379/H6tWrFRKf0qVL8xIBERERZanY3tUVFxcHY2NjhfK2bdvC1NQUbm5u2L9/fyFERkREREVVsejxySwiIgLLly+X6+2RyWRYuHAhXF1doaamht27d8PHxwf79u1D27Zts20rOTkZycnJ4vv4+PgCjZ2IiIgKj0QoxCeljR8/HnPnzs2xTmhoKCpVqiS+f/r0KTw8PODp6Ym//vorx3V79uyJqKgonDt3Lts6/v7+mDp1qkL548ePc5zWnoiIiIqO+Ph4WFpa4v379zA0NMy2XqEmPq9evcKbN29yrGNjYyPeufXs2TN4enrihx9+QGBgINTUcr5St3LlSsyYMQPPnz/Ptk7mHp+nT5+icuXKedgLIiIiKioeP36M8uXLZ7u8UC91mZiYwMTEJFd1nz59ikaNGsHFxQUBAQFfTXoAIDg4GObm5jnWkUqlkEql4nuZTIbHjx9DX1+fDyj8BhmZN3vOqCjheUlFDc9J5REEAR8+fEC5cuVyrFcsxvg8ffoUnp6eqFixIhYsWIBXr16JyzLu4NqwYQO0tLRQs2ZNAMCePXuwfv36r14Oy0xNTS3HTJHyxsDAgL/MVOTwvKSihuekcuR0iStDsUh8jh8/joiICERERCgkJV9eqZs+fToePXoEDQ0NVKpUCdu3b0fHjh2/d7hERERURBXqGB8queLj42FoaIi4uDj+F0NFBs9LKmp4Tn5/xfY5PlS0SaVSTJkyRW78FFFh43lJRQ3Pye+PPT5ERESkMtjjQ0RERCqDiQ8RERGpDCY+REREpDKY+FCW/Pz8IJFIMGfOHLnyffv2yT3YURAE/PHHH6hXrx5kMhmMjIxQu3ZtLFmyBB8/fgQAfPz4ERMmTICtrS20tbVhYmICDw8P/PPPP2I7np6ekEgkCq+BAwd+nx2mYufVq1cYNGgQKlSoAKlUCjMzM3h5eeHChQsAACsrK0gkEmzbtk1h3SpVqkAikSAwMFAss7KywpIlSxTq+vv7o0aNGgW0F6QqYmNjMWzYMNjY2EAqlcLS0hJt2rTByZMn5erNnj0b6urqmD9/fiFFWvIVi+f4UOHQ1tbG3Llz8fPPP6NUqVJZ1unRowf27NmD33//HStWrICJiQlCQkKwZMkSWFlZwcfHBwMHDsTly5exfPlyVK5cGW/evEFQUJDCdCX9+/fHtGnT5Mp0dXULbP+oeOvQoQNSUlKwYcMG2NjY4MWLFzh58qTceWVpaYmAgAB06dJFLLt06RJiY2Ohp6dXGGGTCoqOjoarqyuMjIwwf/58ODs749OnTzh69CiGDBmCsLAwse769esxduxYrF+/HmPGjCnEqEsuJj6UraZNmyIiIgKzZ8/GvHnzFJbv2LEDW7Zswb59+9CuXTux3MrKCm3bthVnut+/fz+WLl2Kli1bistdXFwU2tPV1RWfxE2Uk/fv3+PcuXM4c+YMPDw8AAAVK1ZE3bp15er5+vpi8eLFePz4MSwtLQF8/mLx9fXFxo0bv3vcpJoGDx4MiUSCK1euyCXcVapUQZ8+fcT3Z8+exX///Ydp06Zh48aNCAoKQoMGDQoj5BKNl7ooW+rq6pg1axaWL1+OJ0+eKCzfsmULHB0d5ZKeDBKJRHx0uJmZGQ4dOoQPHz4UeMykGmQyGWQyGfbt2yc3yXBmZcuWhZeXFzZs2ADg82XX7du3y33ZEBWkt2/f4siRIxgyZEiWvYxGRkbiz+vWrUPXrl2hqamJrl27Yt26dd8xUtXBxIdy1L59e9SoUQNTpkxRWBYeHg5HR8evtvHHH38gKCgIpUuXRp06dTBy5EhxHMaXVq1aJX6hZby2bNmilP2gkkVDQwOBgYHYsGEDjIyM4Orqit9++w23bt1SqNunTx8EBgZCEATs2rULtra22Y7ZGTdunMI5OGvWrALeGyrJIiIiIAgCKlWqlGO9+Ph47Nq1C927dwcAdO/eHTt27EBCQsL3CFOlMPGhr5o7dy42bNiA0NBQufLcPvuyYcOGePjwIU6ePImOHTvi7t27cHd3x/Tp0+Xq+fr6Ijg4WO7Vtm1bpe0HlSwdOnTAs2fPsH//fnh7e+PMmTOoVauW3IBlAGjVqhUSEhLw77//Yv369Tn29owZM0bhHOQAe/oWuf07uXXrVtja2qJ69eoAgBo1aqBixYrYvn17QYankpj40Fc1bNgQXl5emDBhgly5g4OD3KC8nGhqasLd3R3jxo3DsWPHMG3aNEyfPh0pKSliHUNDQ9jZ2cm99PX1lbovVLJoa2ujWbNmmDRpEoKCguDn56fQO6mhoYEePXpgypQpuHz5Mnx9fbNtr0yZMgrnoLGxcUHvBpVg9vb2kEgkX/1buW7dOty9excaGhri6969e1i/fv13ilR1MPGhXJkzZw7+97//4eLFi2JZt27d8ODBA7nb0jMIgoC4uLhs26tcuTJSU1ORlJRUIPGSaqpcuTISExMVyvv06YOzZ8+iXbt22d6hSFQQjI2N4eXlhZUrV2Z5br5//x63b9/GtWvXcObMGbnexjNnzuDixYu5/geTcod3dVGuODs7w9fXF8uWLRPLOnfujL1796Jr1674/fff0bx5c5iYmOD27dtYvHgxhg0bBh8fH3h6eqJr166oXbs2SpcujXv37uG3335Do0aN5GYj/vjxI2JjY+W2K5VK+UVFCt68eYNOnTqhT58+qFatGvT19XHt2jXMmzcvy8H2Tk5OeP36NR+PQIVi5cqVcHV1Rd26dTFt2jRUq1YNqampOH78OFavXg0vLy/UrVsXDRs2VFi3Tp06WLduHZ/ro0Ts8aFcmzZtGtLT08X3EokEf//9NxYtWoR9+/bBw8MD1apVg7+/P9q1awcvLy8AEO+qad68OZycnDBs2DB4eXlhx44dcu3/+eefMDc3l3t17dr1u+4jFQ8ymQz16tXD4sWL0bBhQ1StWhWTJk1C//79sWLFiizXKV26NHR0dL5zpESAjY0Nbty4gUaNGmHUqFGoWrUqmjVrhpMnT2Lp0qXYvHkzOnTokOW6HTp0wMaNG/Hp06fvHHXJxdnZiYiISGWwx4eIiIhUBhMfIiIiUhlMfIiIiEhlMPEhIiIilcHEh4iIiFQGEx8iIiJSGUx8iIiISGUw8SEiIiKVwcSHiIiIVAYTHyIqEvz8/CCRSCCRSKCpqYmyZcuiWbNmWL9+vdxUKV8TGBgIIyOjggs0G35+fvDx8fnu2yWivGHiQ0RFhre3N54/f47o6GgcPnwYjRo1wi+//ILWrVsjNTW1sMMjohKAiQ8RFRlSqRRmZmawsLBArVq18Ntvv+Gff/7B4cOHERgYCABYtGgRnJ2doaenB0tLSwwePBgJCQkAgDNnzqB3796Ii4sTe4/8/f0BAJs2bULt2rWhr68PMzMzdOvWDS9fvhS3/e7dO/j6+sLExAQ6Ojqwt7dHQECAuPzx48fo3LkzjIyMYGxsjHbt2iE6OhoA4O/vjw0bNuCff/4Rt3vmzJnvcciIKI+Y+BBRkda4cWNUr14de/bsAQCoqalh2bJluHv3LjZs2IBTp05h7NixAIAGDRpgyZIlMDAwwPPnz/H8+XOMHj0aAPDp0ydMnz4dISEh2LdvH6Kjo+Hn5yduZ9KkSbh37x4OHz6M0NBQrF69GmXKlBHX9fLygr6+Ps6dO4cLFy5AJpPB29sbKSkpGD16NDp37iz2WD1//hwNGjT4vgeKiHJFo7ADICL6mkqVKuHWrVsAgBEjRojlVlZWmDFjBgYOHIhVq1ZBS0sLhoaGkEgkMDMzk2ujT58+4s82NjZYtmwZ6tSpg4SEBMhkMsTExKBmzZqoXbu22HaG7du3Iz09HX/99RckEgkAICAgAEZGRjhz5gyaN28OHR0dJCcnK2yXiIoW9vgQUZEnCIKYcJw4cQJNmjSBhYUF9PX10aNHD7x58wYfP37MsY3r16+jTZs2qFChAvT19eHh4QEAiImJAQAMGjQI27ZtQ40aNTB27FgEBQWJ64aEhCAiIgL6+vqQyWSQyWQwNjZGUlISIiMjC2iviaggMPEhoiIvNDQU1tbWiI6ORuvWrVGtWjXs3r0b169fx8qVKwEAKSkp2a6fmJgILy8vGBgYYMuWLbh69Sr27t0rt16LFi3w6NEjjBw5Es+ePUOTJk3Ey2QJCQlwcXFBcHCw3OvBgwfo1q1bAe89ESkTL3URUZF26tQp3L59GyNHjsT169eRnp6OhQsXQk3t8/9tO3bskKuvpaWFtLQ0ubKwsDC8efMGc+bMgaWlJQDg2rVrCtsyMTFBr1690KtXL7i7u2PMmDFYsGABatWqhe3bt8PU1BQGBgZZxpnVdomo6GGPDxEVGcnJyYiNjcXTp09x48YNzJo1C+3atUPr1q3Rs2dP2NnZ4dOnT1i+fDkePnyITZs2Yc2aNXJtWFlZISEhASdPnsTr16/x8eNHVKhQAVpaWuJ6+/fvx/Tp0+XWmzx5Mv755x9ERETg7t27OHDgAJycnAAAvr6+KFOmDNq1a4dz584hKioKZ86cwfDhw/HkyRNxu7du3cL9+/fx+vVrfPr06fscNCLKG4GIqAjo1auXAEAAIGhoaAgmJiZC06ZNhfXr1wtpaWlivUWLFgnm5uaCjo6O4OXlJWzcuFEAILx7906sM3DgQKF06dICAGHKlCmCIAjC33//LVhZWQlSqVSoX7++sH//fgGAcPPmTUEQBGH69OmCk5OToKOjIxgbGwvt2rUTHj58KLb5/PlzoWfPnkKZMmUEqVQq2NjYCP379xfi4uIEQRCEly9fCs2aNRNkMpkAQDh9+nRBHzIiygeJIAhCYSZeRERERN8LL3URERGRymDiQ0RERCqDiQ8RERGpDCY+REREpDKY+BAREZHKYOJDREREKoOJDxEREakMJj5ERESkMpj4EBERkcpg4kNEREQqg4kPERERqQwmPkRERKQy/h+snsqP9YRIlQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.barplot(data = df_merged, x = 'Dataset', y = 'Difference', hue = 'Model')\n",
    "plt.title(\"Difference between 'Full' and 'Instruct' prompt\\nin terms of Error Reduction Percentage\")\n",
    "\n",
    "plt.savefig(os.path.join(save_appendix,\"prompt_comparison.pdf\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sydney Morning Herald Dataset 1\n",
    "\n",
    "\n",
    "Explore SMH with gpt4 only data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "gpt4_only_configs = pd.DataFrame({\n",
    "    'get_response_func': [get_response_openai], \n",
    "    'engine': ['gpt-4-turbo-preview'\n",
    "                 ],\n",
    "    'rate_limit':[300e3],\n",
    "    'additional_args': [\n",
    "        {}\n",
    "    ]\n",
    "})\n",
    "\n",
    "\n",
    "smh1_articles_raw =  os.path.join(smh_folder, 'dataset1_article_level', 'raw')\n",
    "corrected_folder_smh1 = os.path.join(smh_folder, 'results_dataset1')\n",
    "\n",
    "smh_data1 = files_to_df_core_func(smh1_articles_raw )\n",
    "\n",
    "smh_data1['content'] = smh_data1['content'].str.replace('\\n', ' ')\n",
    "\n",
    "smh_data1['id'] = smh_data1['file_name'] # this is needed as the processing log uses an id to keep track of what has been processed and what hasn't to allow for easy restarts\n",
    "\n",
    "full_prompt_smh = \"{content}\"+f\"\"\" \\n \\n \"\"\" + f\"\"\"You are an expert in post-OCR correction of documents. Using the context available from the text please recover the most likely original text from the corrupted OCR. The text is from The Sydney Morning Herald 1842 -1950. The text may be an advert or article and may be missing the beggining or end. Do not add any text, commentary, or lead in sentences beyond the recovered text. Do not add a title, or any introductions.\"\"\"\n",
    "\n",
    "\n",
    "full_model_configs_smh1 = generate_model_configs(gpt4_only_configs, full_prompt_smh, 'full')\n",
    "\n",
    "compare_request_configurations(smh_data1, full_model_configs_smh1, folder_path=corrected_folder_smh1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'data/overproof/SMH/results_dataset1'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrected_folder_smh1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "No objects to concatenate",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[23], line 8\u001b[0m\n\u001b[1;32m      3\u001b[0m raw_ocr \u001b[38;5;241m=\u001b[39m smh_articles_raw\n\u001b[1;32m      6\u001b[0m smh1_performance_eval \u001b[38;5;241m=\u001b[39m  evaluate_correction_performance_folders(corrected_folder_smh1, gt_folder, wer, cer)\n\u001b[0;32m----> 8\u001b[0m smh1_raw_ocr_eval \u001b[38;5;241m=\u001b[39m  \u001b[43mevaluate_correction_performance\u001b[49m\u001b[43m(\u001b[49m\u001b[43mraw_ocr\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgt_folder\u001b[49m\u001b[43m \u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mraw_ocr\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m     10\u001b[0m smh1_error_reduction \u001b[38;5;241m=\u001b[39m get_metric_error_reduction(smh1_performance_eval, smh_raw_ocr_eval )\n\u001b[1;32m     12\u001b[0m smh1_error_reduction\u001b[38;5;241m.\u001b[39mgroupby(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtype\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;241m.\u001b[39mdescribe()\u001b[38;5;241m.\u001b[39mfilter(regex \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m50|median\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;241m.\u001b[39mround(\u001b[38;5;241m2\u001b[39m)\u001b[38;5;241m.\u001b[39msort_values((eval_metric, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m50\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124m'\u001b[39m))\n",
      "File \u001b[0;32m~/clocrc/evaluation_funcs.py:205\u001b[0m, in \u001b[0;36mevaluate_correction_performance\u001b[0;34m(folder, transcripts_dir, wer_func, cer_func, type, remove_line_breaks)\u001b[0m\n\u001b[1;32m    202\u001b[0m \u001b[38;5;66;03m# filter to only have the ones with a transcript\u001b[39;00m\n\u001b[1;32m    203\u001b[0m dev_data_raw_df \u001b[38;5;241m=\u001b[39m dev_data_raw_df\u001b[38;5;241m.\u001b[39mloc[dev_data_raw_df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfile_name\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39misin(os\u001b[38;5;241m.\u001b[39mlistdir(transcripts_dir))]\u001b[38;5;241m.\u001b[39mreset_index(drop\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m--> 205\u001b[0m eval_temp \u001b[38;5;241m=\u001b[39m \u001b[43mevaluate_ocr_dataframe\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdev_data_raw_df\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtranscripts_dir\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwer_func\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcer_func\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    206\u001b[0m eval_temp[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtype\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mtype\u001b[39m\n\u001b[1;32m    208\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m eval_temp\n",
      "File \u001b[0;32m~/clocrc/evaluation_funcs.py:90\u001b[0m, in \u001b[0;36mevaluate_ocr_dataframe\u001b[0;34m(dev_data_raw_df, dev_transcripts, wer, cer)\u001b[0m\n\u001b[1;32m     87\u001b[0m     results_list\u001b[38;5;241m.\u001b[39mappend(metrics_df)\n\u001b[1;32m     89\u001b[0m \u001b[38;5;66;03m# Combine all the DataFrames in the list into a single DataFrame\u001b[39;00m\n\u001b[0;32m---> 90\u001b[0m combined_df \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconcat\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresults_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mignore_index\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m     92\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m combined_df\n",
      "File \u001b[0;32m~/clocrc/.venv/lib/python3.12/site-packages/pandas/core/reshape/concat.py:382\u001b[0m, in \u001b[0;36mconcat\u001b[0;34m(objs, axis, join, ignore_index, keys, levels, names, verify_integrity, sort, copy)\u001b[0m\n\u001b[1;32m    379\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m copy \u001b[38;5;129;01mand\u001b[39;00m using_copy_on_write():\n\u001b[1;32m    380\u001b[0m     copy \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m--> 382\u001b[0m op \u001b[38;5;241m=\u001b[39m \u001b[43m_Concatenator\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    383\u001b[0m \u001b[43m    \u001b[49m\u001b[43mobjs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    384\u001b[0m \u001b[43m    \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43maxis\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    385\u001b[0m \u001b[43m    \u001b[49m\u001b[43mignore_index\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_index\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    386\u001b[0m \u001b[43m    \u001b[49m\u001b[43mjoin\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    387\u001b[0m \u001b[43m    \u001b[49m\u001b[43mkeys\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkeys\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    388\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlevels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlevels\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    389\u001b[0m \u001b[43m    \u001b[49m\u001b[43mnames\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnames\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    390\u001b[0m \u001b[43m    \u001b[49m\u001b[43mverify_integrity\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mverify_integrity\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    391\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcopy\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcopy\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    392\u001b[0m \u001b[43m    \u001b[49m\u001b[43msort\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msort\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    393\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    395\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m op\u001b[38;5;241m.\u001b[39mget_result()\n",
      "File \u001b[0;32m~/clocrc/.venv/lib/python3.12/site-packages/pandas/core/reshape/concat.py:445\u001b[0m, in \u001b[0;36m_Concatenator.__init__\u001b[0;34m(self, objs, axis, join, keys, levels, names, ignore_index, verify_integrity, copy, sort)\u001b[0m\n\u001b[1;32m    442\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mverify_integrity \u001b[38;5;241m=\u001b[39m verify_integrity\n\u001b[1;32m    443\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcopy \u001b[38;5;241m=\u001b[39m copy\n\u001b[0;32m--> 445\u001b[0m objs, keys \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_clean_keys_and_objs\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobjs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkeys\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    447\u001b[0m \u001b[38;5;66;03m# figure out what our result ndim is going to be\u001b[39;00m\n\u001b[1;32m    448\u001b[0m ndims \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_ndims(objs)\n",
      "File \u001b[0;32m~/clocrc/.venv/lib/python3.12/site-packages/pandas/core/reshape/concat.py:507\u001b[0m, in \u001b[0;36m_Concatenator._clean_keys_and_objs\u001b[0;34m(self, objs, keys)\u001b[0m\n\u001b[1;32m    504\u001b[0m     objs_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(objs)\n\u001b[1;32m    506\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(objs_list) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m--> 507\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNo objects to concatenate\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    509\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m keys \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    510\u001b[0m     objs_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(com\u001b[38;5;241m.\u001b[39mnot_none(\u001b[38;5;241m*\u001b[39mobjs_list))\n",
      "\u001b[0;31mValueError\u001b[0m: No objects to concatenate"
     ]
    }
   ],
   "source": [
    "gt_folder =   os.path.join(smh_folder, 'dataset1_article_level', 'corrected')\n",
    "\n",
    "raw_ocr = smh_articles_raw\n",
    "\n",
    "\n",
    "smh1_performance_eval =  evaluate_correction_performance_folders(corrected_folder_smh1, gt_folder, wer, cer)\n",
    "\n",
    "smh1_raw_ocr_eval =  evaluate_correction_performance(raw_ocr, gt_folder , wer, cer, 'raw_ocr')\n",
    "\n",
    "smh1_error_reduction = get_metric_error_reduction(smh1_performance_eval, smh_raw_ocr_eval )\n",
    "\n",
    "smh1_error_reduction.groupby('type').describe().filter(regex = '50|median').round(2).sort_values((eval_metric, '50%'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation_funcs import load_txt_files_to_df, calculate_metrics\n",
    "\n",
    "dev_data_raw_df = load_txt_files_to_df(os.path.join(corrected_folder_smh1, \n",
    "                                                    \"full__gpt-4-turbo-preview\")).merge(load_txt_files_to_df(gt_folder), on = 'file_name', suffixes=['_results', '_corrected'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content_html_results</th>\n",
       "      <th>file_name</th>\n",
       "      <th>content_html_corrected</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>THE MOVEMENT BY THE STRATHFIELD COUNCIL\\n\\nThe...</td>\n",
       "      <td>14038989_year_1896_type_Article_title_The_Sydn...</td>\n",
       "      <td>THE MOVEMENT BY THE STRATHFIELD\\nCOUNCIL.\\nThe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The greatest difficulty is met with in obtaini...</td>\n",
       "      <td>14037364_year_1896_type_Article_title_The_Sydn...</td>\n",
       "      <td>NAMES OF THE MISSING.\\nThe greatest difficulty...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BANKSTOWN. The fortnightly meeting of this cou...</td>\n",
       "      <td>14070806_year_1896_type_Article_title_The_Sydn...</td>\n",
       "      <td>BANKSTOWN.\\nThe fortnightly meeting of this co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The Hannah Nicholson, though old, is well pres...</td>\n",
       "      <td>14058158_year_1896_type_Article_title_The_Sydn...</td>\n",
       "      <td>THE HANNAH NICHOLSON IN DISTRESS.\\nThe barque,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ARRANGEMENTS FOR THE FUNERAL. SERVICE TO BE HE...</td>\n",
       "      <td>14047181_year_1896_type_Article_title_The_Sydn...</td>\n",
       "      <td>ARRANGEMENTS FOR THE FUNERAL.\\nSERVICE TO BE H...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>542</th>\n",
       "      <td>PASTORAL INTELLIGENCE. (From our CORRESPONDENT...</td>\n",
       "      <td>14067427_year_1896_type_Article_title_The_Sydn...</td>\n",
       "      <td>PASTORAL INTELLIGENCE.\\n(FROM OUR CORRESPONDEN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>543</th>\n",
       "      <td>AQUATICS. The death of Mr. L. M. Dietrich on T...</td>\n",
       "      <td>14060983_year_1896_type_Article_title_The_Sydn...</td>\n",
       "      <td>AQUATICS.\\nTHE LATE MR. E. M. DIETRICH.\\nThe d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544</th>\n",
       "      <td>BREACH OF PROMISE CASE. MELBOURNE, Tuesday. A ...</td>\n",
       "      <td>14064770_year_1896_type_Article_title_The_Sydn...</td>\n",
       "      <td>BREACH OF PROMISE CASE.\\nMELBOURNE, Tuesday.\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>545</th>\n",
       "      <td>WHAT MR. BUSSELL SAYS. SOME PARTICULARS OF THE...</td>\n",
       "      <td>14055213_year_1896_type_Article_title_The_Sydn...</td>\n",
       "      <td>WHAT MR. RUSSELL SAYS.\\nSOME PARTICULARS OF TH...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>546</th>\n",
       "      <td>LIFE-SAVING SOCIETY, MANLY, PENGUIN BRANCH. Mr...</td>\n",
       "      <td>14040001_year_1896_type_Article_title_The_Sydn...</td>\n",
       "      <td>LIFE-SAVING SOCIETY, MANLY, PENGUIN\\nBRANCH.\\n...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>547 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  content_html_results  \\\n",
       "0    THE MOVEMENT BY THE STRATHFIELD COUNCIL\\n\\nThe...   \n",
       "1    The greatest difficulty is met with in obtaini...   \n",
       "2    BANKSTOWN. The fortnightly meeting of this cou...   \n",
       "3    The Hannah Nicholson, though old, is well pres...   \n",
       "4    ARRANGEMENTS FOR THE FUNERAL. SERVICE TO BE HE...   \n",
       "..                                                 ...   \n",
       "542  PASTORAL INTELLIGENCE. (From our CORRESPONDENT...   \n",
       "543  AQUATICS. The death of Mr. L. M. Dietrich on T...   \n",
       "544  BREACH OF PROMISE CASE. MELBOURNE, Tuesday. A ...   \n",
       "545  WHAT MR. BUSSELL SAYS. SOME PARTICULARS OF THE...   \n",
       "546  LIFE-SAVING SOCIETY, MANLY, PENGUIN BRANCH. Mr...   \n",
       "\n",
       "                                             file_name  \\\n",
       "0    14038989_year_1896_type_Article_title_The_Sydn...   \n",
       "1    14037364_year_1896_type_Article_title_The_Sydn...   \n",
       "2    14070806_year_1896_type_Article_title_The_Sydn...   \n",
       "3    14058158_year_1896_type_Article_title_The_Sydn...   \n",
       "4    14047181_year_1896_type_Article_title_The_Sydn...   \n",
       "..                                                 ...   \n",
       "542  14067427_year_1896_type_Article_title_The_Sydn...   \n",
       "543  14060983_year_1896_type_Article_title_The_Sydn...   \n",
       "544  14064770_year_1896_type_Article_title_The_Sydn...   \n",
       "545  14055213_year_1896_type_Article_title_The_Sydn...   \n",
       "546  14040001_year_1896_type_Article_title_The_Sydn...   \n",
       "\n",
       "                                content_html_corrected  \n",
       "0    THE MOVEMENT BY THE STRATHFIELD\\nCOUNCIL.\\nThe...  \n",
       "1    NAMES OF THE MISSING.\\nThe greatest difficulty...  \n",
       "2    BANKSTOWN.\\nThe fortnightly meeting of this co...  \n",
       "3    THE HANNAH NICHOLSON IN DISTRESS.\\nThe barque,...  \n",
       "4    ARRANGEMENTS FOR THE FUNERAL.\\nSERVICE TO BE H...  \n",
       "..                                                 ...  \n",
       "542  PASTORAL INTELLIGENCE.\\n(FROM OUR CORRESPONDEN...  \n",
       "543  AQUATICS.\\nTHE LATE MR. E. M. DIETRICH.\\nThe d...  \n",
       "544  BREACH OF PROMISE CASE.\\nMELBOURNE, Tuesday.\\n...  \n",
       "545  WHAT MR. RUSSELL SAYS.\\nSOME PARTICULARS OF TH...  \n",
       "546  LIFE-SAVING SOCIETY, MANLY, PENGUIN\\nBRANCH.\\n...  \n",
       "\n",
       "[547 rows x 3 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_data_raw_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jellyfish\n",
    "\n",
    "def calculate_metrics(file_name, raw_data_string, contents, wer, cer):\n",
    "    wer_score = wer.compute(predictions=[raw_data_string], references=[contents])\n",
    "    cer_score = cer.compute(predictions=[raw_data_string], references=[contents])\n",
    "    leven_score = jellyfish.levenshtein_distance(raw_data_string, contents)\n",
    "    \n",
    "    # Here we prepare for vectorized computation\n",
    "    #leven_distances = np.array([leven_score])\n",
    "    #lengths = np.array([max(len(raw_data_string), len(contents))])\n",
    "    \n",
    "    #This is not very useful as I can use the error rate reduction\n",
    "    #lev_sim = vectorized_levenshtein_similarity(leven_distances, lengths)[0]  # Get the first element since we're dealing with single values\n",
    "\n",
    "    results_df = {\n",
    "        'File Name': file_name,\n",
    "        'WER': wer_score,\n",
    "        'CER': cer_score,\n",
    "        'lev_dist': leven_score\n",
    "    }\n",
    "\n",
    "    return results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df = dev_data_raw_df.apply(lambda row: calculate_metrics(row['file_name'], row['content_html_results'], row['content_html_corrected'], wer, cer), axis=1, result_type='expand')\n",
    "\n",
    "# If you want to combine the original DataFrame with the new metrics\n",
    "combined_df = pd.concat([dev_data_raw_df, metrics_df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>WER</th>\n",
       "      <th>CER</th>\n",
       "      <th>lev_dist</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>547.000000</td>\n",
       "      <td>547.000000</td>\n",
       "      <td>547.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.370350</td>\n",
       "      <td>0.073910</td>\n",
       "      <td>140.804388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.095112</td>\n",
       "      <td>0.055847</td>\n",
       "      <td>169.815533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.233227</td>\n",
       "      <td>0.022565</td>\n",
       "      <td>18.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.307427</td>\n",
       "      <td>0.043045</td>\n",
       "      <td>60.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.341346</td>\n",
       "      <td>0.057927</td>\n",
       "      <td>96.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.394446</td>\n",
       "      <td>0.083303</td>\n",
       "      <td>165.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.747368</td>\n",
       "      <td>0.546934</td>\n",
       "      <td>2550.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              WER         CER     lev_dist\n",
       "count  547.000000  547.000000   547.000000\n",
       "mean     0.370350    0.073910   140.804388\n",
       "std      0.095112    0.055847   169.815533\n",
       "min      0.233227    0.022565    18.000000\n",
       "25%      0.307427    0.043045    60.000000\n",
       "50%      0.341346    0.057927    96.000000\n",
       "75%      0.394446    0.083303   165.000000\n",
       "max      0.747368    0.546934  2550.000000"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "ncse_correct_compare= ncse_performance_eval.loc[ncse_performance_eval['type']=='full__claude-3-opus-20240229'].merge(ncse_raw_ocr_eval, on = 'File Name', suffixes=[' Corrected', \" OCR\"])\n",
    "ncse_correct_compare['CER_diff'] = ncse_correct_compare['CER OCR'] - ncse_correct_compare['CER Corrected']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>File Name</th>\n",
       "      <th>WER Corrected</th>\n",
       "      <th>CER Corrected</th>\n",
       "      <th>lev_dist Corrected</th>\n",
       "      <th>type Corrected</th>\n",
       "      <th>WER OCR</th>\n",
       "      <th>CER OCR</th>\n",
       "      <th>lev_dist OCR</th>\n",
       "      <th>type OCR</th>\n",
       "      <th>CER_diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>slug_ad01216_periodical_t_issue_ttw_21121867_p...</td>\n",
       "      <td>0.393443</td>\n",
       "      <td>0.265252</td>\n",
       "      <td>102</td>\n",
       "      <td>full__claude-3-opus-20240229</td>\n",
       "      <td>1.245902</td>\n",
       "      <td>0.490716</td>\n",
       "      <td>187</td>\n",
       "      <td>raw_ocr</td>\n",
       "      <td>0.225464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>slug_ar00506_periodical_t_issue_ttw_16051868_p...</td>\n",
       "      <td>0.393617</td>\n",
       "      <td>0.249144</td>\n",
       "      <td>305</td>\n",
       "      <td>full__claude-3-opus-20240229</td>\n",
       "      <td>1.095745</td>\n",
       "      <td>0.452055</td>\n",
       "      <td>538</td>\n",
       "      <td>raw_ocr</td>\n",
       "      <td>0.202911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>slug_ar00801_periodical_pc_issue_tec_01051889_...</td>\n",
       "      <td>0.152542</td>\n",
       "      <td>0.096317</td>\n",
       "      <td>73</td>\n",
       "      <td>full__claude-3-opus-20240229</td>\n",
       "      <td>1.279661</td>\n",
       "      <td>0.481586</td>\n",
       "      <td>343</td>\n",
       "      <td>raw_ocr</td>\n",
       "      <td>0.385269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>slug_ad31203_periodical_pc_issue_tec_06121883_...</td>\n",
       "      <td>0.353659</td>\n",
       "      <td>0.267961</td>\n",
       "      <td>147</td>\n",
       "      <td>full__claude-3-opus-20240229</td>\n",
       "      <td>1.158537</td>\n",
       "      <td>0.464078</td>\n",
       "      <td>244</td>\n",
       "      <td>raw_ocr</td>\n",
       "      <td>0.196117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>slug_ar00500_periodical_ewj_issue_ewj_01081858...</td>\n",
       "      <td>0.466797</td>\n",
       "      <td>0.390486</td>\n",
       "      <td>1189</td>\n",
       "      <td>full__claude-3-opus-20240229</td>\n",
       "      <td>0.787109</td>\n",
       "      <td>0.497192</td>\n",
       "      <td>1510</td>\n",
       "      <td>raw_ocr</td>\n",
       "      <td>0.106706</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            File Name  WER Corrected  \\\n",
       "5   slug_ad01216_periodical_t_issue_ttw_21121867_p...       0.393443   \n",
       "15  slug_ar00506_periodical_t_issue_ttw_16051868_p...       0.393617   \n",
       "47  slug_ar00801_periodical_pc_issue_tec_01051889_...       0.152542   \n",
       "59  slug_ad31203_periodical_pc_issue_tec_06121883_...       0.353659   \n",
       "81  slug_ar00500_periodical_ewj_issue_ewj_01081858...       0.466797   \n",
       "\n",
       "    CER Corrected  lev_dist Corrected                type Corrected   WER OCR  \\\n",
       "5        0.265252                 102  full__claude-3-opus-20240229  1.245902   \n",
       "15       0.249144                 305  full__claude-3-opus-20240229  1.095745   \n",
       "47       0.096317                  73  full__claude-3-opus-20240229  1.279661   \n",
       "59       0.267961                 147  full__claude-3-opus-20240229  1.158537   \n",
       "81       0.390486                1189  full__claude-3-opus-20240229  0.787109   \n",
       "\n",
       "     CER OCR  lev_dist OCR type OCR  CER_diff  \n",
       "5   0.490716           187  raw_ocr  0.225464  \n",
       "15  0.452055           538  raw_ocr  0.202911  \n",
       "47  0.481586           343  raw_ocr  0.385269  \n",
       "59  0.464078           244  raw_ocr  0.196117  \n",
       "81  0.497192          1510  raw_ocr  0.106706  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ncse_correct_compare.loc[ncse_correct_compare['CER OCR'].between(0.4, 0.5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHcCAYAAAAqQ4tyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAACEqElEQVR4nO3dd3xN9/8H8FfWvdmDCEEIEmLFiFK7CLHHj1Jas9TWVqtmbbX3HjVKjRqlRW1qVKuIUrUSIVpEQqaMm/H5/XG+uVz3Jrk3cu9Jbl7Px+M+knvm+5zcm/u+n2khhBAgIiIiMhOWcgdARERElJeY3BAREZFZYXJDREREZoXJDREREZkVJjdERERkVpjcEBERkVlhckNERERmhckNERERmRUmN0RERGRWmNyQWbKwsMDUqVPlDuOtbd26FX5+frCxsYGrq6vJztuvXz94e3vnat+pU6fCwsIibwN6w5kzZ2BhYYEzZ84Y9TxEmR48eAALCwts3rxZ7lBID0xuzFRoaCgGDx6M8uXLw9bWFs7OzmjYsCGWLl2KpKQkucMjPdy+fRv9+vVDhQoVsH79eqxbty7HfS5cuIAuXbqgePHiUCqV8Pb2xuDBgxEeHm6CiPOvmzdv4qOPPkKpUqWgVCpRsmRJfPjhh7h582aW++j7HvL29oaFhYX64eDggLp16+K7774zxaW9lfT0dGzatAnvvfceihQpon7N9O/fH5cvX5Y7vFxZtWpVvkhAwsPDMWTIEHh7e0OpVMLDwwOdO3fGhQsXstwnIiICX375Jfz8/GBvbw8HBwcEBARg5syZiImJUW/33nvvabzm7Ozs4O/vjyVLliAjI8MEV1cACDI7Bw8eFHZ2dsLV1VWMGjVKrFu3TqxYsUJ88MEHwsbGRgwaNEjuEI0uKSlJpKamyh3GW1m9erUAIO7du6fX9suWLRMWFhaiQoUKYsaMGWLDhg3iiy++EC4uLsLFxUVcuHBB73OrVCqRnJycq7hTU1NFUlJSrvbV1+nTpwUAcfr06Ry33bt3r1AoFKJEiRJi4sSJYsOGDWLSpEnC09NTKBQKsW/fPq19DHkPlS1bVtSsWVNs3bpVbN26VcybN09UrFhRABDr1q3Ly8vOU4mJiaJ169YCgGjSpImYP3+++Pbbb8XXX38tKlWqJCwsLMSjR4/kDtNgVatWFU2bNs3z44aFhQkAYtOmTTlue/78eeHs7CycnZ3F6NGjxYYNG8TMmTOFj4+PsLCwEMuWLdPa59KlS8Ld3V3Y2tqKgQMHitWrV4vVq1eLjz/+WDg4OIiWLVuqt23atKkoXbq0+jW3ePFi8c477wgAYsKECXl52QUWkxszc//+feHo6Cj8/PzE48ePtdbfu3dPLFmyRIbIjC89Pd3oH6qmNG3aNAFAREZG5rjt+fPnhaWlpWjcuLF4+fKlxrqQkBBRvHhx4enpKV68eJHtcRISEt4qZlPRN7kJCQkR9vb2ws/PTzx79kxjXWRkpPDz8xMODg4iNDRUvdzQ91DZsmVFu3btNLZ59uyZcHR0FJUrV87F1ZnG8OHDBQCxePFirXVpaWli/vz5eZLcZPeaevO1mhfkTm5evHghSpQoIYoXLy5CQkI01iUmJorGjRsLS0tLjS8b0dHRolSpUqJ48eLi1q1bWsd8+vSpmDFjhvp506ZNRdWqVTW2SUpKEmXLlhVOTk4iLS0tF1doXpjcmJkhQ4YIAHp/S09NTRXTp08X5cuXFwqFQpQtW1aMHz9e61t75j/w06dPi4CAAGFrayuqVaum/nDZu3evqFatmlAqlaJ27dri6tWrGvv37dtX/SHSqlUrYW9vLzw9PcW0adNERkaGxrbz588X9evXF0WKFBG2traidu3aYvfu3VqxAxDDhw8X27ZtE1WqVBHW1tbixx9/VK+bMmWKetu4uDjx6aefirJlywqFQiGKFSsmAgMDxZUrVzSO+cMPP4jatWsLW1tbUbRoUfHhhx+Kf//9V+e1/Pvvv6JTp07CwcFBuLu7iy+++ELvfyorV64UVapUEQqFQnh6eophw4aJ6OhojfsNQOPx+vW8KSgoSFhZWYn79+/rXL9lyxYBQMyePVvrOkJCQkSbNm2Eo6Oj6NSpk3pd2bJlNY4RFRUlPvroI+Hk5CRcXFxEnz59xLVr17T+4U+ZMkW8WSic+bf68ccfRdWqVYVCoRBVqlQRv/zyi8Z2Dx48EEOHDhUVK1YUtra2okiRIqJbt24iLCxMYzt9k5vBgwcLAOLs2bM61//6668CgBg8eLB6maHvIV3JjRBC1KlTRygUimz3ValUws3NTfTr109rXWxsrFAqleKLL75QL1u2bJmoUqWKulQpICBAfP/993rF+bpHjx4Ja2trjdKAnFy9elW0bt1aODk5CQcHB9G8eXNx8eJFjW02bdokAIgzZ86IoUOHimLFiglXV1chxKsP5MuXL4vGjRsLOzs78emnnwohhEhOThaTJ08WFSpUEAqFQpQuXVqMGTNGZ+nh1q1bxTvvvKO+B40bNxZHjx4VQuh+37ye6ERHR4tPP/1UlC5dWigUClGhQgUxZ84ckZ6ernGO6Oho0bdvX+Hs7Kx+rQcHB+uV3MyePVsAEN99953O9ffv3xdWVlYiKChIvWzOnDkCgN5/S13JjRBCdOvWTQDQmZQXNkxuzEypUqVE+fLl9d6+b9++AoDo1q2bWLlypejTp48AIDp37qyxXdmyZUWlSpWEp6enmDp1qli8eLEoVaqUcHR0FNu2bRNlypQRc+bMEXPmzBEuLi7Cx8dH4x9G3759ha2trfD19RW9e/cWK1asEO3btxcAxNdff61xrtKlS4thw4aJFStWiEWLFom6desKAOLgwYMa2wEQlStXFsWKFRPTpk0TK1euFMHBwep1rycDvXr1EgqFQl1EPHfuXNGhQwexbds29TaZ/5jfeecdsXjxYjFu3DhhZ2cnvL29NRKPzGupWrWqGDBggFi9erXo2rWrACBWrVqV4z3P/PAPDAwUy5cvFyNGjBBWVlbinXfeESqVSgghxI8//ii6dOkiAIjVq1eLrVu3ir/++kvn8V6+fCmsra3Fe++9l+U5k5OThVKpFA0bNtS4DqVSKSpUqCD69u0r1qxZo/6H/GZyk56eLurXry+srKzEiBEjxIoVK0TLli1FjRo19E5uatSoITw9PcWMGTPEkiVLRPny5YW9vb2IiopSb7d7925Ro0YNMXnyZLFu3ToxYcIE4ebmJsqWLavxLV/f5KZkyZLC29s72228vb1F6dKl1c8NfQ/pSm5SU1PV395zMmDAAOHq6ipSUlI0lmcmpH/++acQQoh169ap36tr164VS5cuFR9//LEYNWqU3rFmyjxWVh/Ab/r777+Fg4OD+u83Z84cUa5cOaFUKsXvv/+u3i7zPVSlShXRtGlTsXz5cjFnzhwhhPSBXKJECVGsWDExcuRIsXbtWrF//36Rnp6u/sLz2WefibVr14oRI0YIa2trdbKdaerUqQKAaNCggZg/f75YunSp6NWrlxg7dqwQQnrflC5dWvj5+amrbI4dOyaEkN4n/v7+omjRomLChAlizZo1ok+fPsLCwkKdZAkhREZGhmjSpImwtLQUw4YNE8uXLxfNmzcX/v7+eiU3DRo0ELa2ttlW6zZt2lTY2NiIxMRE9T52dnZar4Hs9teV3NSpU0dYWFioj1uYMbkxI7GxsQKA1j+ErGR+6x44cKDG8i+//FIAEKdOnVIvy/xG9Ntvv6mXHT16VAAQdnZ24uHDh+rla9eu1frgyUyiRo4cqV6WkZEh2rVrJxQKhUbVy5tvTJVKJapVqyaaN2+usRyAsLS0FDdv3tS6tjeTGxcXFzF8+PAs74VKpRIeHh6iWrVqGlVbBw8eFADE5MmTta5l+vTpGseoVauWCAgIyPIcQkjVFQqFQrRq1Uoj+VuxYoUAIDZu3Khelpkk5FQtlfl3fP0ftC7+/v6iSJEiWtcxbtw4rW3fTG727t0rAGhUx6Snp4vmzZvrndwoFAqNYvq//vpLABDLly9XL9P1T/nixYtaH8T6JDcxMTF6vR86duwoAIi4uDiD30NCSO+NVq1aicjISBEZGSlu3LghevfurS6tyknm++jnn3/WWN62bVuNJKtTp046P9By4/PPPxcA1F8GctK5c2ehUCg0qu8eP34snJycRJMmTdTLMpObRo0aaZViNm3aVAAQa9as0Vi+detWYWlpKc6dO6exfM2aNRolaPfu3ROWlpaiS5cuWiUtr5f+ZlUtNWPGDOHg4CDu3r2rsXzcuHHCyspKhIeHCyGE2L9/vwAg5s2bp94mLS1NNG7cWK/kxtXVVdSoUSPbbUaNGiUAiOvXrwshhHBzc8txn9c1bdpU+Pn5qV9zt2/fFmPGjBEAdJYiFkbsLWVG4uLiAABOTk56bX/48GEAwOjRozWWf/HFFwCAQ4cOaSyvUqUK6tevr35er149AEDz5s1RpkwZreX379/XOueIESPUv1tYWGDEiBFQqVQ4ceKEermdnZ369+joaMTGxqJx48a4evWq1vGaNm2KKlWq5HClgKurK/744w88fvxY5/rLly/j2bNnGDZsGGxtbdXL27VrBz8/P617AQBDhgzReN64cWOd1/y6EydOQKVS4bPPPoOl5au336BBg+Ds7KzzPDmJj48HkPPf3cnJSf0aed3QoUNzPMeRI0dgY2ODQYMGqZdZWlpi+PDhescZGBiIChUqqJ/7+/vD2dlZ4569/rdPTU3F8+fP4ePjA1dXV51//+wYcl8A6f1j6Hso07Fjx1CsWDEUK1YM1atXx9atW9G/f3/Mnz8/x32bN28Od3d37Nq1S70sOjoax48fR48ePdTLXF1d8e+//+LPP/80KDZdDLnO9PR0HDt2DJ07d0b58uXVyz09PdGrVy+cP39e63U1aNAgWFlZaR1LqVSif//+Gst2796NypUrw8/PD1FRUepH8+bNAQCnT58GAOzfvx8ZGRmYPHmyxnsHgF5DD+zevRuNGzeGm5ubxnkCAwORnp6Os2fPApD+L1pbW2u8L6ysrDBy5MgczwFIrztDXnOZPw19zd2+fVv9mvPz88P8+fPRsWPHfNFTLD9gcmNGnJ2dAbz6p56Thw8fwtLSEj4+PhrLS5QoAVdXVzx8+FBj+esJDAC4uLgAALy8vHQuj46O1lhuaWmp8c8RACpWrAhAGkMi08GDB/Huu+/C1tYWRYoUQbFixbB69WrExsZqXUO5cuVyukwAwLx58/D333/Dy8sLdevWxdSpUzU+VDOvtVKlSlr7+vn5ad0LW1tbFCtWTGOZm5ub1jW/KavzKBQKlC9fXus8+sj8p5jT313XP11ra2uULl06x3M8fPgQnp6esLe311j+5msnO2++fgDte5aUlITJkyfDy8sLSqUS7u7uKFasGGJiYnT+/bNjyH3J3N7Q91CmevXq4fjx4zhy5AgWLFgAV1dXREdHQ6FQ5LivtbU1unbtigMHDiAlJQUAsG/fPqSmpmokN2PHjoWjoyPq1q0LX19fDB8+PNtuxdkx5DojIyORmJio871RuXJlZGRk4NGjRxrLs3pflipVSuue3Lt3Dzdv3lR/UGc+Mv83PHv2DIDUNd/S0lKvLzO63Lt3D0eOHNE6T2BgoMZ5Ml/rjo6OGvvrun5dnJycDHrNAdLfw9DXnLe3N44fP46jR49i1apVKFWqFCIjIzW+nBVm1nIHQHnH2dkZJUuWxN9//23QfvoOuKbrm1h2y4UQBsUBAOfOnUPHjh3RpEkTrFq1Cp6enrCxscGmTZuwfft2re1f/6afne7du6Nx48b48ccfcezYMcyfPx9z587Fvn370KZNG4PjzOqa5eDj4wNra2tcv349y21SUlJw584d1KlTR2O5UqnU+hZsLPq8TkaOHIlNmzbhs88+Q/369eHi4gILCwt88MEHBo/f4eLiAk9Pz2zvCwBcv34dpUqVUn/g5+Y95O7urv6QDAoKgp+fH9q3b4+lS5dqlYzq8sEHH2Dt2rX45Zdf0LlzZ/zwww/w8/NDjRo11NtUrlwZd+7cwcGDB3HkyBHs3bsXq1atwuTJkzFt2jSD4vXz8wMA3LhxAzVr1jRoX31k9b7UtTwjIwPVq1fHokWLdO7z5pen3MrIyEDLli3x1Vdf6VyfmUy9rcqVKyM4OBgpKSlQKpU6t7l+/TpsbGzg6+sLQPp7XLt2DSqVSq+EGAAcHBzUrzkAaNiwIWrXro0JEyZg2bJlb38hBRxLbsxM+/btERoaiosXL+a4bdmyZZGRkYF79+5pLI+IiEBMTAzKli2bp7FlZGRoVdvcvXsXANSj4e7duxe2trY4evQoBgwYgDZt2mi8gd+Gp6cnhg0bhv379yMsLAxFixbFrFmzAEB9rXfu3NHa786dO3l2L7I6j0qlQlhYWK7O4+DggGbNmuHs2bNZlvz88MMPSElJQfv27Q0PGlLcT548QWJiosbykJCQXB0vK3v27EHfvn2xcOFCdOvWDS1btkSjRo00BjAzRPv27REWFobz58/rXH/u3Dk8ePBA474Y8h7KSrt27dC0aVN88803ePnyZY7bN2nSBJ6enti1axeioqJw6tQpjVKbTA4ODujRowc2bdqE8PBwtGvXDrNmzUJycrJB8bVp0wZWVlbYtm1bjtsWK1YM9vb2Ot8bt2/fhqWl5VslIBUqVMCLFy/QokULBAYGaj0yS0wqVKiAjIwM/PPPP9keL6svaxUqVEBCQoLOcwQGBqpLFjNf6wkJCRr767p+Xdq3b4/k5GTs3r1b5/oHDx7g3LlzaN68uTrZ69ChA5KSkrB37169zqGLv78/PvroI6xdu7bQD9oJMLkxO1999RUcHBwwcOBAREREaK0PDQ3F0qVLAQBt27YFACxZskRjm8xvUO3atcvz+FasWKH+XQiBFStWwMbGBi1atAAgfbu3sLBAenq6ersHDx5g//79uT5nenq6VpWGh4cHSpYsqa4GqFOnDjw8PLBmzRr1MgD45ZdfcOvWrTy7F4GBgVAoFFi2bJlGicW3336L2NjYXJ9n0qRJEEKgX79+WiNQh4WF4auvvoKnpycGDx6cq+MHBQUhNTUV69evVy/LyMjAypUrc3W8rFhZWWmV+C1fvlzj9WCIMWPGwM7ODoMHD8bz58811r148QJDhgyBvb09xowZo15uyHsoO2PHjsXz58817llWLC0t0a1bN/z888/YunUr0tLStJKbN+NXKBSoUqUKhBBITU0FACQmJuL27duIiorK9nxeXl4YNGgQjh07huXLl2utz8jIwMKFC/Hvv//CysoKrVq1woEDBzSqjyMiIrB9+3Y0atRIXeqVG927d8d///2n8z4lJSWpk8POnTvD0tIS06dP1yrFe/014+DgoDMZ7t69Oy5evIijR49qrYuJiUFaWhoA6f9iWloaVq9erV6fnp6u8z7pMnjwYHh4eGDMmDFaX+aSk5PRv39/CCEwefJk9fIhQ4bA09MTX3zxhfoL3+uePXuGmTNn5njur776CqmpqVmWghUmrJYyMxUqVMD27dvRo0cPVK5cGX369EG1atWgUqnw22+/Yffu3ejXrx8AoEaNGujbty/WrVuHmJgYNG3aFJcuXcKWLVvQuXNnNGvWLE9js7W1xZEjR9C3b1/Uq1cPv/zyCw4dOoQJEyao26+0a9cOixYtQuvWrdGrVy88e/YMK1euhI+PT47VC1mJj49H6dKl0a1bN9SoUQOOjo44ceIE/vzzTyxcuBAAYGNjg7lz56J///5o2rQpevbsiYiICCxduhTe3t74/PPP8+QeFCtWDOPHj8e0adPQunVrdOzYEXfu3MGqVavwzjvv4KOPPsrVcZs0aYIFCxZg9OjR8Pf3R79+/eDp6Ynbt29j/fr1yMjIwOHDh+Hm5par43fu3Bl169bFF198gZCQEPj5+eGnn37CixcvAOhftZmT9u3bY+vWrXBxcUGVKlVw8eJFnDhxAkWLFs3V8Xx9fbFlyxZ8+OGHqF69Oj7++GOUK1cODx48wLfffouoqCjs2LFDo6GzIe+h7LRp0wbVqlXDokWLMHz4cNjY2GS7fY8ePbB8+XJMmTIF1atXR+XKlTXWt2rVCiVKlEDDhg1RvHhx3Lp1CytWrEC7du3UbTcuXbqEZs2aYcqUKTnOrbZw4UKEhoZi1KhR2LdvH9q3bw83NzeEh4dj9+7duH37Nj744AMAwMyZM3H8+HE0atQIw4YNg7W1NdauXYuUlBTMmzcvx3uRnd69e+OHH37AkCFDcPr0aTRs2BDp6em4ffs2fvjhBxw9ehR16tSBj48PJk6ciBkzZqBx48b4v//7PyiVSvz5558oWbIkZs+eDQAICAjA6tWrMXPmTPj4+MDDwwPNmzfHmDFj8NNPP6F9+/bo168fAgIC8PLlS9y4cQN79uzBgwcP4O7ujg4dOqBhw4YYN24cHjx4gCpVqmDfvn16t/kqWrQo9uzZg3bt2qF27doYOHAgqlSpgqdPn2Lz5s0ICQnB0qVL0aBBA/U+bm5u+PHHH9G2bVvUrFkTH330EQICAgAAV69exY4dOzQ6c2SlSpUqaNu2LTZs2ICvv/461+8bsyBbPy0yqrt374pBgwYJb29voVAohJOTk2jYsKFYvny5xvgLqampYtq0aaJcuXLCxsZGeHl5ZTuI35ugo7tr5kie8+fPVy/TNYhf8eLFxZQpU7S6dX777bfC19dXKJVK4efnJzZt2pTtwHC64LWu4CkpKWLMmDGiRo0a6gHIatSooXNMml27dolatWoJpVIpihQpku0gfm/SFWNWVqxYIfz8/ISNjY0oXry4GDp0qMZYOq8fT58RijOdPXtWdOrUSbi7uwsbGxtRpkwZMWjQIPHgwQOtbbO6jsx1bw7iFxkZKXr16qUexK9fv37iwoULAoDYuXOnVtyvy+pvVbZsWdG3b1/18+joaNG/f3/h7u4uHB0dRVBQkLh9+7bWdoZMvyCEENevXxc9e/YUnp6ewsbGRpQoUUL07NlT3LhxI8t99H0PZfXeEEKIzZs36z1kf0ZGhvDy8hIAxMyZM7XWr127VjRp0kQULVpUPT7RmDFjRGxsrHqbzPuS3YCPr0tLSxMbNmwQjRs3Fi4uLsLGxkaULVtW9O/fX6ub+NWrV0VQUJBwdHQU9vb2olmzZhpDQwjxqit45tg8r8tqbBYhpKEY5s6dK6pWrSqUSqVwc3MTAQEBYtq0aRrXJ4QQGzduVL9H3dzcRNOmTcXx48fV658+fSratWsnnJyctAbxi4+PF+PHjxc+Pj5CoVAId3d30aBBA7FgwQL1GFNCCPH8+XPRu3dv9SB+vXv31nsQv0xhYWFi0KBBokyZMsLGxka4u7uLjh07anV5f93jx4/F559/rh7E0t7eXgQEBIhZs2Zp3Ifs7uWZM2cMeg2YKwshctHqk8hA/fr1w549e7Tqsalg279/P7p06YLz58+jYcOGcodDRASAbW6ISE9vtuXJbIfg7OyM2rVryxQVEZE2trkhIr2MHDkSSUlJqF+/PlJSUrBv3z789ttv+Oabb/Tukk9EZApMbohIL82bN8fChQtx8OBBJCcnw8fHB8uXL9cYdZqIKD9gmxsiIiIyK2xzQ0RERGaFyQ0RERGZFSY3ZDYiIiLQrVs3FC1aFBYWFlojL8tp6tSpeTbQnaE2b94MCwsLjdFlKW88ePAAFhYWGjMxy/m3LkgsLCxyHGiQKLeY3JDZ+Pzzz3H06FGMHz8eW7duRevWrY1+zuTkZCxevBj16tWDi4sLbG1tUbFiRYwYMULnMOrmJDw8HEOGDIG3tzeUSiU8PDzQuXPnXM9UbSpnzpyBhYWFzkfmiLym0K9fP62Zp/PS4cOHZU8e8kMMVDixtxSZjVOnTqFTp0748ssvTXK+qKgotG7dGleuXEH79u3Rq1cvODo64s6dO9i5cyfWrVsHlUplklhM7cKFC+q5yd4cXr5x48ZYunQpRo4cKXOU2Rs1ahTeeecdjWWZE7iag8OHD2PlypWyJhfZxZCUlARra34EkXHwlUVm49mzZ3B1dc2z4yUnJ0OhUMDSUncBZ79+/RAcHIw9e/aga9euGutmzJiBiRMn5lks+Ul0dDS6desGOzs7XLhwQWNeptGjRyMoKAifffYZAgICNObPyW8aN26Mbt26yR1GvpCWloaMjAwoFAqTndPW1tZk56LCh9VSVOBltikRQmDlypXqKoZM9+/fx/vvv48iRYrA3t4e7777Lg4dOqRxjMyqip07d2LSpEkoVaoU7O3tERcXp/Ocf/zxBw4dOoSPP/5YK7EBAKVSiQULFmQb96ZNm9C8eXN4eHhAqVSiSpUqGjMRZ8qqbYK3t7fWBI43b95E8+bNYWdnh9KlS2PmzJlaMyhn+uWXX9C4cWM4ODjAyckJ7dq1w82bN7ONGQDWrl2Lp0+fYv78+RqJDQDY2dlhy5YtsLCwwPTp09XLM/9GZ8+exeDBg1G0aFE4OzujT58+iI6OztX1pqamYtq0afD19YWtrS2KFi2KRo0a4fjx4zleQ0503VsAeO+99/Dee++99fGzO2/79u1x/vx51K1bF7a2tihfvjy+++47je1yuvZ+/fqpZ2x/vdoNeNVOaMGCBViyZAkqVKgApVKJf/75J8v2WZnvjzNnzmgs/+OPP9C2bVu4ubnBwcEB/v7+6hnTs4shc9mbf+fg4GC0adMGzs7OcHR0RIsWLfD7779rbJMZ44ULFzB69GgUK1YMDg4O6NKlCyIjIzW2vXz5MoKCguDu7g47OzuUK1cOAwYM0POvQQUZS26owGvSpAm2bt2K3r17o2XLlujTp496XUREBBo0aIDExESMGjUKRYsWxZYtW9CxY0fs2bMHXbp00TjWjBkzoFAo8OWXXyIlJSXLb7I//fQTAGlG49xavXo1qlatio4dO8La2ho///wzhg0bhoyMDAwfPtzg4z19+hTNmjVDWloaxo0bBwcHB6xbt07n6MFbt25F3759ERQUhLlz5yIxMRGrV69Go0aNEBwcnG31zM8//wxbW1t0795d5/py5cqhUaNGOHXqFJKSkjTOP2LECLi6umLq1Km4c+cOVq9ejYcPH6o/PA0xdepUzJ49GwMHDkTdunURFxeHy5cv4+rVq2jZsmWO+8fHxyMqKkpjWZEiRbIsqTOVkJAQdOvWDR9//DH69u2LjRs3qmexrlq1KoCcr33w4MF4/Pgxjh8/jq1bt+o8z6ZNm5CcnIxPPvkESqUSRYoUMSjO48ePo3379vD09MSnn36KEiVK4NatWzh48CA+/fRTvWJ43c2bN9G4cWM4Ozvjq6++go2NDdauXYv33nsPv/76K+rVq6ex/ciRI+Hm5oYpU6bgwYMHWLJkCUaMGIFdu3YBkEpyW7VqhWLFimHcuHFwdXXFgwcPsG/fPoOukwooWaftJMpD0DHz9GeffSYAaMzEGx8fL8qVKye8vb3VM5JnzqZcvnx5kZiYmOO5unTpIgBozeSdFV0zZes6T1BQkChfvrzWdema4ffNmbIzr/WPP/5QL3v27JlwcXERAERYWJgQQrp+V1dXMWjQII3jPX36VLi4uGgtf5Orq6uoUaNGttuMGjVKABDXr18XQryaLTogIEBj9uV58+YJAOLAgQMGX2+NGjWynI07O5l/a12PzHv05rkyNW3aVGOW6bCwMK2ZovWdHV7XrOxly5YVAMTZs2fVy549eyaUSqX44osv1Mv0ufbhw4frjCMzZmdnZ/Hs2TONdZl/p8z7kOnNWdjT0tJEuXLlRNmyZbXeAxkZGTnGIIT237lz585CoVCI0NBQ9bLHjx8LJycn0aRJE60YAwMDNc71+eefCysrKxETEyOEEOLHH3/McoZyMn+sliKzdvjwYdStWxeNGjVSL3N0dMQnn3yCBw8e4J9//tHYvm/fvnrNk5RZXeXk5JTr2F4/T2xsLKKiotC0aVPcv38fsbGxBh/v8OHDePfdd1G3bl31smLFiuHDDz/U2O748eOIiYlBz549ERUVpX5YWVmhXr16OH36dLbniY+Pz/G6M9e/Wa33ySefwMbGRv186NChsLa2xuHDh/W6xte5urri5s2buHfvnsH7AsDkyZNx/PhxjUeJEiVyday8VKVKFTRu3Fj9vFixYqhUqRLu37+vXva21w4AXbt2RbFixXK1b3BwMMLCwvDZZ59ptXPLTTf49PR0HDt2DJ07d0b58uXVyz09PdGrVy+cP39e52vp9XM1btwY6enpePjwIQCo4zp48CBSU1MNjokKNiY3ZNYePnyISpUqaS2vXLmyev3rypUrp9dxnZ2dAUgf9Ll14cIFBAYGwsHBAa6urihWrBgmTJgAALlKbh4+fAhfX1+t5W9ef+YHYvPmzVGsWDGNx7Fjx/Ds2bNsz+Pk5JTjdWeufzMJejM+R0dHeHp65moMnunTpyMmJgYVK1ZE9erVMWbMGFy/fl3v/atXr47AwECNR35o5FqmTBmtZW5ubhptk9722gH9X+u6hIaGAgCqVauW62O8LjIyEomJiVm+VzMyMvDo0SON5W/eJzc3NwBQ36emTZuia9eumDZtGtzd3dGpUyds2rQJKSkpeRIz5W9Mboheo+/s1n5+fgCAGzdu5Oo8oaGhaNGiBaKiorBo0SIcOnQIx48fx+effw4AWTYCfl16enquzp157K1bt2qVXBw/fhwHDhzIdv/KlSvjzp072X5IXL9+HTY2NjqTrdx683qbNGmC0NBQbNy4EdWqVcOGDRtQu3ZtbNiw4a3PlVXpQ27vuSGsrKx0LhevTQOYF9eu67Uu53UbKqf7ZGFhgT179uDixYsYMWIE/vvvPwwYMAABAQFISEgwZagkAyY3ZNbKli2LO3fuaC2/ffu2en1udOjQAQCwbdu2XO3/888/IyUlBT/99BMGDx6Mtm3bIjAwUOcHjpubG2JiYjSWqVQqPHnyRGNZ2bJldVZTvHn9mT2cPDw8tEouAgMDc+wN1L59eyQnJ2P37t061z948ADnzp1T99p63ZvxJSQk4MmTJxoNmPW9XkBqANy/f3/s2LEDjx49gr+/f56M66IrBkC7pE9OOV17bqqHMks/3rz2N6878zX0999/Z3s8fWMoVqwY7O3ts3yvWlpawsvLS69jvendd9/FrFmzcPnyZXz//fe4efMmdu7cmatjUcHB5IbMWtu2bXHp0iVcvHhRvezly5dYt24dvL29UaVKlVwdt379+mjdujU2bNiA/fv3a61XqVTZDiaY+a3z9W/jsbGx2LRpk9a2FSpUwNmzZzWWrVu3TuvbdNu2bfH777/j0qVL6mWRkZH4/vvvNbYLCgqCs7MzvvnmG51tEd7sTvumwYMHw8PDA2PGjNFoBwJIYwP1798fQghMnjxZa99169ZpnHP16tVIS0tDmzZtDL7e58+fazx3dHSEj49PnlQ7VKhQAb///rvGIIwHDx7UqhqRiz7X7uDgAEA7UclOZtLy+v1PT0/HunXrNLarXbs2ypUrhyVLlmgd//XXtL4xWFlZoVWrVjhw4IBGFWVERAS2b9+ORo0aqauC9RUdHa0RCwDUrFkTAFg1VQiwKziZtXHjxmHHjh1o06YNRo0ahSJFimDLli0ICwvD3r1736rb73fffYdWrVrh//7v/9ChQwe0aNECDg4OuHfvHnbu3IknT55kOdZNq1atoFAo0KFDBwwePBgJCQlYv349PDw8tEooBg4ciCFDhqBr165o2bIl/vrrLxw9ehTu7u4a23311VfqaSc+/fRTdVfwsmXLarTHcHZ2xurVq9G7d2/Url0bH3zwAYoVK4bw8HAcOnQIDRs2xIoVK7K87qJFi2LPnj1o164dateurTVCcUhICJYuXapzAD+VSoUWLVqge/fuuHPnDlatWoVGjRqhY8eOBl9vlSpV8N577yEgIABFihTB5cuXsWfPHowYMSLrP5qeBg4ciD179qB169bo3r07QkNDsW3bNq1xfeSiz7UHBAQAkEZiDgoKgpWVVY7TS1StWhXvvvsuxo8fjxcvXqBIkSLYuXMn0tLSNLaztLTE6tWr0aFDB9SsWRP9+/eHp6cnbt++jZs3b+Lo0aMGxzBz5kwcP34cjRo1wrBhw2BtbY21a9ciJSUF8+bNM/gebdmyBatWrUKXLl1QoUIFxMfHY/369XB2dlaPrk1mTM6uWkR5CTq6ggshRGhoqOjWrZtwdXUVtra2om7duuLgwYMa22R2dd29e7dB50xMTBQLFiwQ77zzjnB0dBQKhUL4+vqKkSNHipCQEPV2uroH//TTT8Lf31/Y2toKb29vMXfuXLFx40atrrjp6eli7Nixwt3dXdjb24ugoCAREhKis7vy9evXRdOmTYWtra0oVaqUmDFjhvj222+z7N4bFBQkXFxchK2trahQoYLo16+fuHz5sl7XHhYWJgYNGiTKlCkjbGxshLu7u+jYsaNGt/tMmd13f/31V/HJJ58INzc34ejoKD788EPx/PlzjW31vd6ZM2eKunXrCldXV2FnZyf8/PzErFmzNLqa66Lv33rhwoWiVKlSQqlUioYNG4rLly+bpCu4ri7eb55Xn2tPS0sTI0eOFMWKFRMWFhbqmDJjnj9/vs64QkNDRWBgoFAqlaJ48eJiwoQJ4vjx4xpdwTOdP39etGzZUjg5OQkHBwfh7+8vli9fnmMMQuju8n/16lURFBQkHB0dhb29vWjWrJn47bffNLbJfC292cX7ze7qV69eFT179hRlypQRSqVSeHh4iPbt2+v9+qaCzUKIN8rtiIjy2ObNm9G/f3/8+eefqFOnjtzhEJGZY5sbIiIiMitMboiIiMisMLkhIiIis8I2N0RERGRWWHJDREREZoXJDREREZmVQjeIX0ZGBh4/fgwnJ6dcDU9OREREpieEQHx8PEqWLJnjAKyFLrl5/PhxrucoISIiInk9evQIpUuXznabQpfcODk5AZBujqFzlRARkZl5/BgoWVLuKEgPcXFx8PLyUn+OZ6fQJTeZVVHOzs5MboiICpOkJGDGDGDiROB/k3qCnwMFjj5NSgpdckNERIVQcjLQpQtw9Cjw11/AoUNyR0RGxN5SRERk3lJSgK5dpcTG3h4YO1buiMjImNwQEZH5SkkBunUDDh8G7OykEpsmTeSOioyMyQ0REZknlQro3h04eBCwtQV+/hl47z25oyITYHJDRETmacgQ4KefAKVS+tmihdwRkYkwuSEiIvM0ahRQqhRw4ADQsqXc0ZAJsbcUERGZp5o1gZAQqUqKChWW3BARkXlISwMGDQIuXHi1jIlNocTkhoiICr70dKBvX2DDBqBDByA2Vu6ISEasliIiooItPR3o3x/Yvh2wtgY2bgRcXOSOqlCKTVQhKkGFuORUONvZwN1BARd7hcnjkLXk5uzZs+jQoQNKliwJCwsL7N+/P8d9zpw5g9q1a0OpVMLHxwebN282epxERJRPZWQAAwcCW7cCVlbAzp1A585yR1UoPY5JwogdwWix6Fd0WfUbWiz8FSN3BONxTJLJY5E1uXn58iVq1KiBlStX6rV9WFgY2rVrh2bNmuHatWv47LPPMHDgQBw9etTIkRIRUb6TkSG1sdm8WUpsduyQRiImk4tNVGHs3us4dy9KY/nZe1EYt/c6YhNVJo1H1mqpNm3aoE2bNnpvv2bNGpQrVw4LFy4EAFSuXBnnz5/H4sWLERQUZKwwiYgoP1q9WqqCsrQEvv8eeP99uSMqtKISVFqJTaaz96IQlaAyafVUgWpQfPHiRQQGBmosCwoKwsWLF2WKiIiIZDNwINCpk1Ql1aOH3NEUanHJqdmuj89hfV4rUA2Knz59iuLFi2ssK168OOLi4pCUlAQ7OzutfVJSUpCSkqJ+HhcXZ/Q4iYjISISQflpYSCMP//ij9DvJytnWJtv1Tjmsz2sFquQmN2bPng0XFxf1w8vLS+6QiIgoN4QAPvsMGD1aM8kh2bk7KtDE113nuia+7nB3NG2PqQKV3JQoUQIREREayyIiIuDs7Kyz1AYAxo8fj9jYWPXj0aNHpgiViIjykhDAF18Ay5YBS5YAv/8ud0T0Ghd7BeZ09ddKcJr4umNuV3+TdwcvUNVS9evXx+HDhzWWHT9+HPXr189yH6VSCaVSaezQiIjIWIQAvvoKWLxYer5+PZDN/32SR0lXOyzvWQtRCSrEJ6fCydYG7o7yjHMja3KTkJCAkJAQ9fOwsDBcu3YNRYoUQZkyZTB+/Hj8999/+O677wAAQ4YMwYoVK/DVV19hwIABOHXqFH744QccOnRIrksgIiJjEgIYPx5YsEB6vmaN1JCY8iUXe3mSmTfJWi11+fJl1KpVC7Vq1QIAjB49GrVq1cLkyZMBAE+ePEF4eLh6+3LlyuHQoUM4fvw4atSogYULF2LDhg3sBk5EZI6EACZNAubOlZ6vXAkMHixvTFQgWAiR2SqrcIiLi4OLiwtiY2Ph7OwsdzhERJSVv/4CateWButbvhwYMULuiEhGhnx+F6g2N0REVIjUqCGNYRMZycSGDMLkhoiI8peXLwEHB+n3Xr3kjYUKpALVFZyIiMzc7NlSVdR//8kdCRVgTG6IiCh/mDcPmDABuHsXOHhQ7mioAGNyQ0RE8lu4EBg7Vvp95kz2iqK3wuSGiIjktXgx8OWX0u/TpgETJ8obDxV4TG6IiEg+y5ZJc0UBwOTJ0oPoLTG5ISIieSQlAStWSL9PnAhMnSprOGQ+2BWciIjkYWcHnD4N7NoFfP45Z/imPMOSGyIiMq2wsFe/lyolVUsxsaE8xOSGiIhM59tvgYoVgR075I6EzBiTGyIiMo3Nm4FBg4C0NODyZbmjITPG5IaIiIzvu++AAQOkmb5HjgQWLJA7IjJjTG6IiMi4vv8e6NdPSmyGDQOWLmUbGzIqJjdERGQ8O3YAffpIic3gwcDy5UxsyOiY3BARkfFcugRkZAADBwKrVgGW/Ngh4+M4N0REZDyLFgH16gHduzOxIZPhK42IiPLWhQuASiX9bmEBfPABExsyKb7aiIgo7xw4ALz3HtCtG5CSInc0VEgxuSEiorzx88/A++9L49g4OgLWbPlA8mByQ0REb+/wYam0JjUV6NFDGtfGykruqKiQYnJDRERv58gRoEsXqZ3N++8D27ax1IZkxeSGiIhy79gxoHNnKbHp2lUasI+JDcmMr0AiIso9OzspmWnTRhqwz8ZG7oiImNwQEdFbaNwY+O03wM+PiQ3lG6yWIiIiw5w9C/z116vn/v6AQiFfPERvYHJDRET6O3dOqoJq3hy4fVvuaIh0YnJDRET6uXBBSmwSE4E6dQBvb7kjItKJyQ0REeXs4kWgdWvg5UsgMBDYvx+wtZU7KiKdmNwQEVH2/vgDCAoCEhKk6qgDB6ReUkT5FJMbIiLK2vXrQKtWQHy8NGfUTz8B9vZyR0WULXYFJyKirJUrB9SoIf1+8CDg4CBvPER6YHJDRERZc3KS5o0SgokNFRisliIiIk1//QUsXPjquaOjlOQQFRAsuSEioleuXwdatACePwdcXYGPP5Y7IiKDseSGiIgkf//9KrF55x2gWze5IyLKFSY3REQE/POP1M07KgoICJBm+3ZxkTsqolxhckNEVNjduiUlNpGRQK1aUmLj6ip3VES5xuSGiKgwi42VqqIiIqQu38ePA0WKyB0V0VthckNEpIfYRBVCnyUgODwaoZEJiE1UyR1S3nBxASZOlBKbEyeAokXljojorVkIIYTcQZhSXFwcXFxcEBsbC2dnZ7nDIaIC4HFMEsbuvY5z96LUy5r4umNOV3+UdDWTaQhUKkChkDsKoiwZ8vnNkhsiomzEJqq0EhsAOHsvCuP2Xi+YJTihoUCnTsCLF6+WMbEhM8LkhogoG1EJKq3EJtPZe1GISihgyU1YGNCsmTRH1LBhckdDZBRMboiIshGXnJrt+vgc1ucrDx9Kic2jR0DFisDixXJHRGQUTG6IiLLhbGuT7XqnHNbnG+HhUmLz8CHg6wucPg14esodFZFRMLkhIsqGu6MCTXzdda5r4usOd8cC0Fbl33+lxCYsDKhQQUpsSpaUOyoio2FyQ0SUDRd7BeZ09ddKcJr4umNuV3+42BeA5KZ3b+D+faB8eSmxKVVK7oiIjIpdwYmI9BCbqEJUggrxyalwsrWBu6OiYCQ2AHDvnjQB5rZtQJkyckdDlCuGfH5zVnAiIj242BegZAYAMjIAy/8Vzvv6Ar/+ClhYyBsTkYmwWoqIyNw8ewbUqQMcPvxqGRMbKkSY3BARmZPISGkSzOBgYNQoaeRhokKGyQ0RkbmIipImwbx5U+oN9csvHHmYCiUmN0RE5uD5cyAwELhxAyhRAjh1SmprQ1QIMbkhIiroXrwAWrYE/voLKF5c6u5dqZLcURHJhskNEVFBt3y51MbGw0MqsfHzkzsiIlmxKzgRUUE3aZJUejNoEFClitzREMmOyQ0RUUGUkADY2QFWVtJj6VK5IyLKN1gtRURU0MTHA61aAf37A+npckdDlO+w5IaIqCCJjwfatAEuXgRu3wYePJAmwyQiNZbcEBEVFAkJQLt2wIULgKsrcPw4ExsiHWRPblauXAlvb2/Y2tqiXr16uHTpUrbbL1myBJUqVYKdnR28vLzw+eefIzk52UTREhHJ5OVLoH174Nw5wMUFOHYMCAiQOyqifEnW5GbXrl0YPXo0pkyZgqtXr6JGjRoICgrCs2fPdG6/fft2jBs3DlOmTMGtW7fw7bffYteuXZgwYYKJIyciMqHERKBDB2nyS2dn4OhR4J135I6KKN+SNblZtGgRBg0ahP79+6NKlSpYs2YN7O3tsXHjRp3b//bbb2jYsCF69eoFb29vtGrVCj179syxtIeIqEALDpaqopycpMSmXj25IyLK12RLblQqFa5cuYLAwMBXwVhaIjAwEBcvXtS5T4MGDXDlyhV1MnP//n0cPnwYbdu2zfI8KSkpiIuL03gQERUoDRsC+/cDR44A774rdzRE+Z5svaWioqKQnp6O4sWLaywvXrw4bt++rXOfXr16ISoqCo0aNYIQAmlpaRgyZEi21VKzZ8/GtGnT8jR2IiKjS04Gnj0DypSRnrdpI288RAWI7A2KDXHmzBl88803WLVqFa5evYp9+/bh0KFDmDFjRpb7jB8/HrGxserHo0ePTBgxEVEupKQAXbsCDRoAISFyR0NU4MhWcuPu7g4rKytERERoLI+IiECJEiV07vP111+jd+/eGDhwIACgevXqePnyJT755BNMnDgRlpbauZpSqYRSqcz7CyAiMgaVCnj/feDwYWkE4v/+A3x85I6KqECRreRGoVAgICAAJ0+eVC/LyMjAyZMnUb9+fZ37JCYmaiUwVlZWAAAhhPGCJSIyBZUK6N4d+PlnwNZW+tm0qdxRERU4so5QPHr0aPTt2xd16tRB3bp1sWTJErx8+RL9+/cHAPTp0welSpXC7NmzAQAdOnTAokWLUKtWLdSrVw8hISH4+uuv0aFDB3WSQ0RUIKWmAh98ABw4ACiVwE8/AS1ayB0VUYEka3LTo0cPREZGYvLkyXj69Clq1qyJI0eOqBsZh4eHa5TUTJo0CRYWFpg0aRL+++8/FCtWDB06dMCsWbPkugQioreXmgr06gX8+COgUEg9o1q2lDsqogLLQhSy+py4uDi4uLggNjYWzs7OcodDRATExADNmgH//CMlONkMb0FUWBny+c2JM4mI5ObqCpw8CVy7BjRvLnc0RAVegeoKTkRkNtLTgRMnXj0vUoSJDVEeYXJDRGRq6enAgAFSu5qVK+WOhsjsMLkhIjKljAxg4EDgu+8AKysgi3G9iCj3mNwQEZlKRgbwySfA5s1SYrN9uzQSMRHlKSY3RESmkJEBDB0KfPstYGkJbNsmDdhHRHmOyQ0RkbEJAYwYAaxbJyU2W7dKA/YRkVEwuSEiMjYLC6B4cennli3SgH1EZDQc54aIyBSmTAG6dAH8/eWOhMjsseSGiEiH2EQVQp8lIDg8GqGRCYhNVBl2ACGA1auBhIRXy5jYEJkES26IiN7wOCYJY/dex7l7UeplTXzdMaerP0q62uV8ACGAr74CFiwAdu4ETp2SekcRkUmw5IaI6DWxiSqtxAYAzt6Lwri913MuwRECGD9eSmwAoGdPJjZEJsbkhojoNVEJKq3EJtPZe1GISsgmuRECmDQJmDtXer5iBTBkiBGiJKLsMLkhInpNXHJqtuvjs1s/ZQrwzTfS78uWAcOH52FkRKQvJjdERK9xtrXJdr1TVuvnzQNmzJB+X7QIGDkyjyMjIn0xuSEieo27owJNfN11rmvi6w53R4XuHVu1AooWBebPBz7/3IgRElFOmNwQEb3GxV6BOV39tRKcJr7umNvVHy72WSQ3NWsCt28DX35p/CCJKFvsCk5E9IaSrnZY3rMWohJUiE9OhZOtDdwdFdqJzeLFQN26QMOG0nN33SU+RGRaTG6IiHRwsdeRzLxu4UKplMbBAfjnH6BMGdMFR0TZYrUUEZGhFi9+Vf00ZgwTG6J8hskNEZEhli0DRo+Wfv/6a6n7NxHlK0xuiIj0tXIl8Omn0u8TJgDTpskbDxHpxOSGiEgfhw4BI0ZIv48dC8ycCVhYyBsTEenEBsVERPoIDAQ6dAAqVQJmz2ZiQ5SPMbkhItKHUgns2ydNgsnEhihfY7UUEVFWNm+WGg8LIT23tmZiQ1QAsOSGiEiX774DBgyQEpsGDYBu3eSOiIj0xJIbIqI3ff890K+flNgMHQp07Sp3RERkACY3RESv27ED6NNHSmw++QRYsYJVUUQFDJMbIqJMP/wAfPQRkJEBDBwIrF4NWPLfJFFBw3ctEREA/Psv0Lu3lNj07w+sXcvEhqiAYoNiIiIAKF1a6h114gSwbh0TG6ICzEKIzD6OhUNcXBxcXFwQGxsLZ2dnucMhIrmlpUldvIkoXzPk85tfTYio8Dp4EKhVC/jvP7kjIaI8xOSGiAqnw4elLt5//w0sXix3NESUh5jcEFHhc/Qo8H//B6hU0uB8c+bIHRER5SEmN0RUuBw/DnTqBKSkAF26ANu3s80NkZlhckNEhcfJk0DHjlJi06kTsHMnYGMjd1RElMeY3BBR4ZCRAXzxBZCcDLRvLw3Yp1DIHRURGQGTGyIqHCwtpUbEw4cDe/YwsSEyY0xuiMi8RUe/+r1kSWmuKKVSvniIyOj0akW3bNkyvQ84atSoXAdDRJSnLlwA2rUDVq0CevWSOxoiMhG9RiguV66cxvPIyEgkJibC1dUVABATEwN7e3t4eHjg/v37Rgk0r3CEYqJC4vffgVatgPh46eeRI5zdm6gAy/MRisPCwtSPWbNmoWbNmrh16xZevHiBFy9e4NatW6hduzZmzJiRJxdARPRWLl0CgoKkxKZZM+DHH5nYEBUiBs8tVaFCBezZswe1atXSWH7lyhV069YNYWFheRpgXmPJDZGZu3wZCAwEYmOBpk2BQ4cABwe5oyKit2TUuaWePHmCtLQ0reXp6emIiIgw9HBERHnn6lWgZUspsWncWJo7iokNUaFjcHLTokULDB48GFevXlUvu3LlCoYOHYrAwMA8DY6IyCB79wIxMUDDhlKJjaOj3BERkQwMTm42btyIEiVKoE6dOlAqlVAqlahbty6KFy+ODRs2GCNGIiL9zJwJLFsG/PIL4OQkdzREJBOD29xkunv3Lm7fvg0A8PPzQ8WKFfM0MGNhmxsiMxMSAnh5cewaIjNnyOd3rmeL8/b2hhACFSpUgDUnnSMiOdy8KfWGqldPGnWYCQ4RIRfVUomJifj4449hb2+PqlWrIjw8HAAwcuRIzJkzJ88DJCLS6Z9/gObNgchI4MkTIClJ7oiIKJ8wOLkZP348/vrrL5w5cwa2trbq5YGBgdi1a1eeBkdEpNPt21Ji8+wZUKsWcOwY8L9BRYmIDK5P2r9/P3bt2oV3330XFq8NilW1alWEhobmaXBERFru3pUSm4gIoEYN4PhxoEgRuaMionzE4JKbyMhIeHh4aC1/+fKlRrJDRJTn7t2T2tg8eQJUrw6cOAEULSp3VESUzxic3NSpUweHDh1SP89MaDZs2ID69evnXWRERG969gyIiwOqVgVOngTc3eWOiIjyIYOrpb755hu0adMG//zzD9LS0rB06VL8888/+O233/Drr78aI0YiIknDhlJpjbc3UKyY3NEQUT5lcMlNo0aNcO3aNaSlpaF69eo4duwYPDw8cPHiRQQEBBgjRiIqzB48AK5ff/W8Xj2geHHZwiGi/C/Xg/gVVBzEj6gAefgQeO89qSrq1CmpATERFUpGnTjTysoKz54901r+/PlzWFlZGXo4IiLdHj2SGg8/eCD1hmL7GiLSk8HJTVYFPSkpKVAoFAYHsHLlSnh7e8PW1hb16tXDpUuXst0+JiYGw4cPh6enJ5RKJSpWrIjDhw8bfF4iysf+/VdKbMLCgAoVgNOngVKl5I6KiAoIvRsUL1u2DIDUO2rDhg1wfG223fT0dJw9exZ+fn4GnXzXrl0YPXo01qxZg3r16mHJkiUICgrCnTt3dHY3V6lUaNmyJTw8PLBnzx6UKlUKDx8+hCsH7yIyH48fS+PYhIYC5ctLiU3p0nJHRUQFiN5tbsqVKwcAePjwIUqXLq1RBaVQKODt7Y3p06ejXr16ep+8Xr16eOedd7BixQoAQEZGBry8vDBy5EiMGzdOa/s1a9Zg/vz5uH37NmxsbPQ+z+vY5oYoH4uIAJo2Be7ckXpE/forUKaM3FERUT5glIkzw8LCAADNmjXDvn374Obm9lZBqlQqXLlyBePHj1cvs7S0RGBgIC5evKhzn59++gn169fH8OHDceDAARQrVgy9evXC2LFj2d6HyBw4OEg9oZKSpBIbJjZElAsGj3Nz+vTpPDlxVFQU0tPTUfyNLp3FixfH7du3de5z//59nDp1Ch9++CEOHz6MkJAQDBs2DKmpqZgyZYrOfVJSUpCSkqJ+HhcXlyfxE5ERODoChw8Dz58zsSGiXDO4QXHXrl0xd+5creXz5s3D+++/nydBZSUjIwMeHh5Yt24dAgIC0KNHD0ycOBFr1qzJcp/Zs2fDxcVF/fDy8jJqjERkoMhIYO3aV88dHJjYENFbMTi5OXv2LNq2bau1vE2bNjh79qzex3F3d4eVlRUiIiI0lkdERKBEiRI69/H09ETFihU1qqAqV66Mp0+fQqVS6dxn/PjxiI2NVT8ePXqkd4xEZGRRUUBgIDBkCLBwodzREJGZMDi5SUhI0Nnl28bGxqAqH4VCgYCAAJw8eVK9LCMjAydPnsxyjqqGDRsiJCQEGRkZ6mV3796Fp6dnlt3QlUolnJ2dNR5ElA88fy4lNtevAyVKAO3byx0REZkJg5Ob6tWrY9euXVrLd+7ciSpVqhh0rNGjR2P9+vXYsmULbt26haFDh+Lly5fo378/AKBPnz4aDY6HDh2KFy9e4NNPP8Xdu3dx6NAhfPPNNxg+fLihl0FEcnrxAmjZEvjrL6kB8enTQKVKckdFRGbC4AbFX3/9Nf7v//4PoaGhaN68OQDg5MmT2LFjB3bv3m3QsXr06IHIyEhMnjwZT58+Rc2aNXHkyBF1I+Pw8HBYWr7Kv7y8vHD06FF8/vnn8Pf3R6lSpfDpp59i7Nixhl4GEcklOhpo1QoIDpYmvzx1CjBwjCwiouzkam6pzBKTa9euwc7ODv7+/pgyZQqaNm1qjBjzFMe5IZJRairQqBFw6ZI0ncLp00C1anJHRUQFgFHGuXldu3bt0K5du1wFR0SFmI0N0KcPcP8+cPIkExsiMgqD29wA0vxOGzZswIQJE/DixQsAwNWrV/Hff//laXBEZIaGDwfu3gX8/eWOhIjMlMHJzfXr11GxYkXMnTsX8+fPR0xMDABg3759Go1/iYgAAPHxUlfv/30RAgC85QjnRETZMTi5GT16NPr164d79+7B1tZWvbxt27YGjXNDRIVAQgLQrp00SF/XroDhTfwMFpuoQuizBASHRyM0MgGxibrHwCIi82Vwm5s///wTa18fTfR/SpUqhadPn+ZJUERkBl6+lMauOXcOcHYG5s0DLCyMdrrYRBWexiXj3+gkWFhY4Gp4NDaeD0Odsm6Y09UfJV3tjHZuIspfDE5ulEqlzsH67t69i2LFiuVJUERUwCUmAh06SLN6OzkBx44B77xjtNM9jknC2D3XcS4kSr2soU9RLOtZC6N2BGPc3utY3rMWXOx1D/ZJRObF4Gqpjh07Yvr06UhNTQUAWFhYIDw8HGPHjkXXrl3zPEAiKmCSkoBOnaRu3o6OwJEjQL16RjtdbKIKY/dqJjYAcCHkOTZdCMOARuVw9l4UohJYPUVUWBic3CxcuBAJCQnw8PBAUlISmjZtCh8fHzg5OWHWrFnGiJGICpKhQ4ETJ6QJMH/5BWjQwKini0pQ4dy9KJ3rLoQ8Ry0vVwBAfHKqUeMgovzD4GopFxcXHD9+HBcuXMBff/2FhIQE1K5dG4GBgcaIj4gKmkmTgD/+kBoRN2pk9NPF5ZC0pKRJc9E52drk2TljE1WISlAhLjkVznY2cHdQsMqLKB8xKLlJTU2FnZ0drl27hoYNG6Jhw4bGiouICiofH+DGDcA6V2OEGsw5h6RFaW2JJr7ucHfMm+TjcUySVA32WmlRE193NlomykcMqpaysbFBmTJlkJ6ebqx4iKigUamA7t2Bw4dfLTNRYgMA7o4KNPF117muoU9RPItPwdyu/nlSsqJu3/NGNdjZe1EYt/c6u50T5RMGt7mZOHGixsjERFSIpaYCPXoAu3cDvXoB/xvU05Rc7BWY09VfK8Fp7OuOWZ2ro221EvDMoxKV7Nr3sNEyUf5h8NerFStWICQkBCVLlkTZsmXh4OCgsf7q1at5FhwR5WOpqcAHHwD79wNKJfDDD4CrqyyhlHS1w/KetRCVoEJ8ciqcbG3g7pj37WByat/DRstE+YPByU3nzp2NEAYRFSipqVJJzb59gEIhJTitWskakou98Rv15tS+Jy8bLRNR7hmU3KSlpcHCwgIDBgxA6dKljRUTEeVnaWnARx8Be/ZIs3zv2we0bi13VCaR2b7nrI6qqbxstExEb8egNjfW1taYP38+0tLSjBUPEeV3GzZIVVA2NsDevdLcUYVEVu17mvi651mjZSJ6ewZXSzVv3hy//vorvL29jRAOEeV7gwYBV65I80Z16CB3NCZnqvY9RJR7Bic3bdq0wbhx43Djxg0EBARoNSju2LFjngVHRPlERoY0o7eVlfRYv17uiGRlivY9RJR7FkIIYcgOlpZZ12RZWFjk+zFw4uLi4OLigtjYWDg7O8sdDlH+l5EBDBwoNSLevFlKboiITMyQz2+DS24yMjJyHRgRFTAZGcDgwcCmTYClpTRvlJHniiIielsGD+JHRIVERgYwbJjUgNjSEti2jYkNERUIuUpufv31V3To0AE+Pj7w8fFBx44dce7cubyOjYjkIgQwYoQ0+aWFBbBlC9Czp9xRmURsogqhzxIQHB6N0MgETqlAVAAZXC21bds29O/fH//3f/+HUaNGAQAuXLiAFi1aYPPmzejVq1eeB0lEJiQEMGoUsHq1lNhs3iyNa1MIcFJMIvNgcIPiypUr45NPPsHnn3+usXzRokVYv349bt26lacB5jU2KCbKwc2bQO3aUgPib78F+veXOyKTiE1UYcSOYJ1zRzXxdcfynrXYQ4pIRoZ8fhtcLXX//n100DG2RceOHREWFmbo4Ygov6laVZpOYf36QpPYAJwUk8icGFwt5eXlhZMnT8LHx0dj+YkTJ+Dl5ZVngRGRCQkBPH8OuP9v5N02beSNRwacFJPIfBic3HzxxRcYNWoUrl27hgb/6zlx4cIFbN68GUuXLs3zAInIyIQAJkwAtm4FTp8GfH1lCSM2UYWoBBXiklPhbGcDdwfTDpTHSTGJzIfByc3QoUNRokQJLFy4ED/88AMAqR3Orl270KlTpzwPkIiMSAjg66+BOXOk52fPypLc5IeGvJwUk8h8GNyguKBjg2Ki10yZAkyfLv2+dKnUS8rE8lND3scxSRi397pGgpM5KaYne0sRycooIxRHR0dj27Zt6Nu3r9ZBY2Nj8d133+lcR0T51PTprxKbRYtkSWwA/Rrymiq54aSYROZB795SK1aswNmzZ3UmLy4uLjh37hyWL1+ep8ERkZHMmiWV2gDA/PnAG0M7mFJ+a8jrYq9ABQ9H1CzjhgoejkxsiAogvZObvXv3YsiQIVmuHzx4MPbs2ZMnQRGRESUnA/v2Sb/PmQN8+aWs4bAhLxHlNb2Tm9DQUPhm09DQ19cXoaGheRIUERmRrS1w4oQ0QN/YsXJHo27Iqwsb8hJRbuid3FhZWeHx48dZrn/8+DEsLTkPJ1G+9ddfr353cwMGDJAvlte42Cswp6u/VoKT2ZCX1UJEZCi9GxTXqlUL+/fvx7vvvqtz/Y8//ohatWrlWWBElIeWLJHa1SxbBowcKXc0WtiQl4jykt7JzYgRI/DBBx+gdOnSGDp0KKysrAAA6enpWLVqFRYvXozt27cbLVAiyqXly181GI6MlDeWbLjYM5khorxh0Dg3EydOxOzZs+Hk5ITy5csDkOaaSkhIwJgxYzAncyCwfIzj3FChsmoVMHy49PuECcDMmdJM30REBYwhn98GD+J36dIlfP/99wgJCYEQAhUrVkSvXr1Qt27dtwraVJjcUKGxdi2Q2cPxq6+knlFMbIiogDLKIH6Z6tatW2ASGaJCa/36V4nNF18wsSGiQoXdm4jM0dOn0s/PPpMG6WNiQ0SFiMElN0RUAHz9NVCvHtCyJRMbIip0WHJDZC5++QVISHj1vFUrJjZEVCgxuSEyB9u3A+3bA23bAomJckdDRCSrPEtukpOTsWDBgrw6HBHpa+dOoHdvICMD8POTplcgIirEDEpuIiMjcfDgQRw7dgzp6ekAgNTUVCxduhTe3t4FYpwbIrOyezfw0UdSYvPxx8CaNQCnQSGiQk7vBsXnz59H+/btERcXBwsLC9SpUwebNm1C586dYW1tjalTp6Jv377GjJWIXrdvH9CzJ5CeDvTrB6xbx8SGiAgGlNxMmjQJbdu2xfXr1zF69Gj8+eef6NKlC7755hv8888/GDJkCOzs7IwZKxFl+uknoEcPKbHp3RvYsIGJDRHR/+g9QnHRokVx7tw5VKlSBUlJSXB0dMS+ffvQqVMnY8eYpzhCMZmFv/4CWrQAWrcGtmwB/jfXGxGRuTLKCMXR0dFwd3cHANjZ2cHe3h7VqlV7u0iJKHdq1AAuXQLKlGFiQ0T0BoMG8fvnn3/w9H8jnwohcOfOHbx8+VJjG39//7yLjoheOXIEcHQEGjWSnv9v8loiItKkd7WUpaUlLCwsoGvzzOUWFhbqXlT5FaulqEA6ehTo1AmwtgZ++w3glwgiKmSMUi0VFhb21oERUS4cPw507gykpABt2gCVK8sdERFRvqZ3clO2bFljxkFEupw8CXTsCCQnSz937QJsbOSOiogoX9O77+i8efOQlJSkfn7hwgWkpKSon8fHx2PYsGF5Gx1RYXb6NNChg5TYtGsH/PADoFDIHZXsYhNVCH2WgODwaIRGJiA2UVWgjk9Exqd3mxsrKys8efIEHh4eAABnZ2dcu3YN5f/XqDEiIgIlS5ZkmxuivHDtGtCwoTRPVJs2wI8/Akql3FHJ7nFMEsbuvY5z96LUy5r4umNOV3+UdH37cbaMfXwiyj1DPr/1Lrl5MwfSMyciotyoXBkIDASCgqSRiJnYIDZRpZV4AMDZe1EYt/f6W5ewGPv4RGQ6BnUFJyITUSqleaPS0zkR5v9EJai0Eo9MZ+9FISpBBRf73FfbGfv4RGQ6HK+dKL/4/Xdg/Hggs1RUoQAKyJQmpminEpecmu36+BzWy318IjIdg0puNmzYAEdHRwBAWloaNm/erB61OD4+Pu+jIyosLl2SqqDi4oBSpYARI+SOSG+maqfibKvZS8xeYYUBjcqhlpcrUtIyYKuwQmxi7ktX3jz+m5xyWE9E+YfeDYq9vb1hYWGR43a5GQ9n5cqVmD9/Pp4+fYoaNWpg+fLlqFu3bo777dy5Ez179kSnTp2wf/9+vc7FBsWU71y+LLWviY0FmjQBDh8GHBzkjkovsYkqjNgRrLM6p4mvO5b3rJVnVTmxiSqM3BGMs/eiYK+wwrKetbDpQhguhDzXOGduk6rXj/+mvL4WIjKcIZ/feic3xrJr1y706dMHa9asQb169bBkyRLs3r0bd+7cUffM0uXBgwdo1KgRypcvjyJFijC5oYLp6lVpAsyYGGlahV9+kaZYMJGHDx+iRWBLnDp5AmXKlDF4/9BnCWix6Ncs158c3RQVPPLueh7HJGHc3uvw93JFcHi0RmKT6W0Skczjn32jFGpuV394srcUkayMMkKxsSxatAiDBg1C//79AQBr1qzBoUOHsHHjRowbN07nPunp6fjwww8xbdo0nDt3DjExMSaMmCiPXLsmldjExAANGkglNiZMbABg+sxZuH//PqbPnIUN69YavL+p26mUdLXD8p618CQ2GStOhejc5m0a/2YePypBhfjkVDjZ2sDdUWHWJTaxiSpEJagQl5wKZzsbuDuY9/VS4aB3g+JTp06hSpUqiIuL01oXGxuLqlWr4uzZswadXKVS4cqVKwgMDHwVkKUlAgMDcfHixSz3mz59Ojw8PPDxxx8bdD6ifCM+XmpjEx0NvPuuVGLj5GTSEB4+fIjvtmyBe8ex+G7LFoSHhxu0f2yiCnY2Vlj1YW1s7PcORjT3gb1Cc4ZyY7RTcbFXICk1+/G03iapcrFXoIKHI2qWcUMFD0ez/qB/HJOEETuC0WLRr+iy6je0WPgrRu4IxuOYpJx3JsrH9E5ulixZgkGDBuksCnJxccHgwYOxePFig04eFRWF9PR0FC9eXGN58eLF1bOPv+n8+fP49ttvsX79er3OkZKSgri4OI0HkeycnIAlS6QSmyNHABmqSL+eOgMOlRrAvlID2Fesj6+nztB738wPxdZLz2HY91cxYPOfCA6PxrKetdQJThNfd7g7GicxYOPfnOXUg43j+pA50zu5+euvv9C6dess17dq1QpXrlzJk6CyEh8fj969e2P9+vXqXlo5mT17NlxcXNQPLy8vo8ZIlK3Xm7j17AmcOwe4uJg8jEvX72Dbtu/gUK8HAMChXg9s2/od/rxxN8d9s/pQvBDyHJsuhGFAo3LqdirGKvVwd1Sgia/u/wHGTKoKCn1KZPQZ14eooNI7uYmIiIBNNhP2WVtbIzIy0qCTu7u7w8rKChEREVrnKlGihNb2oaGhePDgATp06ABra2tYW1vju+++w08//QRra2uEhoZq7TN+/HjExsaqH48ePTIoRqI8c+sW0LQp8N9/r5ZZmn6oqdhEFT4cOQ52vg1g4y4l+zbuXrD1rY9eI8fl+I09uw/FCyHP0b66J5b3rGXUBrgu9grM6eqvleAYO6kqCPQtkeG4PmTO9G5QXKpUKfz999/w8fHRuf769evw9PQ06OQKhQIBAQE4efIkOnfuDADIyMjAyZMnMULHOB9+fn64ceOGxrJJkyYhPj4eS5cu1Vkqo1QqoeTQ9SS3O3eA5s2Bp0+BTz8F9uyRLZS/boUg5MIhlOy3TGO5a4MPELrlU1y/HYrGtStnuX9OH4rJqekmSS4KY+Nffeg70jKr9sic6Z3ctG3bFl9//TVat24N2zeGg09KSsKUKVPQvn17gwMYPXo0+vbtizp16qBu3bpYsmQJXr58qe491adPH5QqVQqzZ8+Gra0tqlWrprG/q6srAGgtJ8o37t4FmjWTEht/f2Ct4b2S8tKihfNgX/FVqU0mG3cv2PnWx+KF89D4+01Z7p+fPhRd7JnMvEnfEpnMqr2sxvUp7FV7VLDpXSY+adIkvHjxAhUrVsS8efNw4MABHDhwAHPnzkWlSpXw4sULTJw40eAAevTogQULFmDy5MmoWbMmrl27hiNHjqgbGYeHh+PJkycGH5coXwgJkRKbJ0+AatWAEyeAokVlC+fhw4c4tGcHXBt8oHO9a4MP8POeHdn2nGJ7l/xN3+STVXtkzgwaxO/hw4cYOnQojh49qp4V3MLCAkFBQVi5ciXKlStntEDzCgfxI5MJDQXeew/491+gShXg9Gkgm4EpTeHjQZ9g3+/34NLuyyy3iT20AF3rV8x23JvCNNhdQRsHxtCRljOvj1V7lN8ZfYTi6OhohISEQAgBX19fuLm55TpYU2NyQybTsqVUUlO5spTYvDHkgak9fPgQPr4V4dFnqVaV1OtSox4hcutnCLl3N9tRiwvDh6Kp5s3Ka4Up+aTCo0BNv2BqTG7IZJ48AYYOBdasAXT0/jO1jwd9gn1/3INL26xLbTLFHl6Aru9mX3pj7kw5b5YxFIbkkwoXQz6/Td8PlcicpaS8+t3TE9i/P18kNpmjEdvX7aHX9vZ1e+Rq1GJzUtDHgSlMIy0TvYnJDVFeefQIqF4d2LZN7ki0TJ85C3Zl/WGhsENaXFSODwuFHWzLVMf0mbPkDj1HOY3Em1scB4ao4JJ94kwis/Dff1KvqNBQYPp04P33gXw0vtKJEycR/+A+4kP6GbTf8bQXxgkojxizTUx+6vJORIZhyQ3R23r8+FViU66c1Ig4HyU2APAwLBRCCIMfD8O0R/3OL4w9NxK7vBMVXExuiN7GkyfSyMP37gFly0q9orLpYUR5x9htYjgODFHBxWopotyKiJASmzt3pITm9GkpwaFcM2RMGVO0ieEUD0QFE5MbotzasgW4fRsoXVpKbArAIJb5maHtZ0zVJoZTPBAVPKyWIsqtMWOkxsOnTwPly8sdTYGWm/YzbBNDRFlhckNkiBcvXo1lY2EBfP014OMjb0xmIDftZ9gmhoiywmopIn29eAG0aAGUKgXs3ZvvekQVZLltP8M2MUSkC5MbIn1ER0tzRV27JvWQ+u8/VkW9pdcbD9sprDCiuQ82ng9Doipda9vs2s+wTQwRvYnJDVFOYmKAVq2Aq1eBYsWAU6eY2LwlXY2HG/kUxbKetTBqR7BGgtPIpyhsbViDTkT6438MouzExgJBQcDly4C7u5TYVKkid1QFWlaNh8+HPMfmC2EY0OhVr7OGPkXRr2E5TP3pZp5Nq0BE5o8lN0RZiYsDWrcGLl0CihYFTp4EqlWTO6oCL7vGw+dDnmNsGz9U8XSG0toSwY9i1CU5UQkqVj8RkV6Y3BBl5d494O+/gSJFpCkV/P3ljsgs5NR4+NGLJAz7/qrWck5USUT6YnJDlJWAAODoUcDWFqhZU+5ozEZOg+8prXXXlnOiSiLSF9vcEL3u5Uvg1q1Xzxs0AGrXli8eM5Td4HuNfIoi+FGM1nIOykdEhmByQ5QpMRHo0AFo1AgIDpY7GrOV3eB733SpjjtP4rSWc1A+IjIEq6WIACApCejYUZpKwckJULFnTm7oO/FldoPvLXi/BgflI6K3wuSGKCkJ6NRJ6g3l6AgcOQLUqyd3VAWOoRNfZjX4HgflI6K3xWopKtySk4EuXYDjxwEHB+CXX6R2NmSQ3Ex8SURkLExuqPBKSQH+7/+kHlH29sDhw1J7GzJYbia+JCIyFlZLUeGVliY1IrazAw4dApo0kTuiAiu3E1+aO33bIBFR3mJyQ4WXg4OU1Ny8CdStK3c0BVpOY9cUxjFqDG2DRER5h9VSVLikpgI//PDquYMDE5s8kN3YNYVxjBq2QSKSF5MbKjxSU4EPPgB69ABmzJA7GrOS3dg1hXGMGrZBIpIXq6WocEhLAz78ENi3D1AogDp15I7I7GQ3dk1h8Hr7mrQMke22hbUNEpGpMLkh85eWBvTuDezeDdjYAHv3Am3ayB2VWSqsY9S82b7m277ZJ8+FsQ0SkSmxWorMW3o60LcvsHOnlNjs2QO0by93VGRGdLWvCX4Ug4Y+RXVuXxjbIBGZGpMbMl9CAAMGANu3A9bWUkPijh3ljorMjK72NRvPh6F/w3JaCU5hbYNEZGqsliLzZWEhTaOwY4dUctO5s9wRUR7KL2PI6BrjJ1GVjlE7gjGgUTlMbFsZqrSMQtcGiUhOTG7IvA0bJrWvKVdO7kgoD+kaQ6axrztmdKoGN3sbkyYQWY3xk6hKx4pTIehSsxSqlHQxWTxExGopMjcZGcDs2cDz56+WMbExK1mNIXPuXhQm7r+Bw38/xeOYJJPFwzF+iPIfJjdkPoQAhg8HJkwAWrWSekmR2cluDJkLIc/h4aQ06UB5HOOHKP9htRSZByGAESOANWuktjaffy41Ii5E8ksbFGPLaR6rlLQM9UB5prr+gjzGT2F53VDhUrj++5N5EgIYNQpYtUpKbDZtAj76SO6oTMqQeYwK+odZTvNYKa2lAmlTD5RXEMf44fxXZK5YLUUFmxBSKc2KFdLzDRukcW3ykdhEFUKfJSA4PBqhkQl5Xl1iyDxGj2OSMGJHMFos+hVdVv2GFgt/xcgdwSZto/K2smvj0tCnKIIfxQDgQHk54fxXZM6Y3FDBNmsWsHSp9Pv69dK4NvmIKZIJfecxMpcPs6zauDT0KYr+Dcth4/kwNuTVA+e/InPGaikq2Hr2lJKaCROAgQPljkZDTsnE8p618qQaI6c2KJnVM/p8mBWUapXMNi5P45Lxb7SUKAY/isGoHcGoU9aNDXn1oO/rhqggYnJDBVuFCsA//wAODnJHosVUyURObVAyq2fM7cMss41LCWdbRCWoUNRBgS41SxWYhrxy0/d1Q1QQsVqKChYhgKlTgYMHXy3Lh4kNYLpkQt9xVsz1w8zFXoEKHo6oWcYNFTwcmdjoiePzkDljckMFy9SpwLRpQNeuQFiY3NFky9jJRGZD5QfPX2J6p2o5jrPCDzN6HcfnIXPGaikqOKZPlx6ANApxPh95ODOZOKujauptk4k3u/DaK6zwdfsqmNiuMpJU6TrHWcn8MBu397pGTPwwK7wK8vg8RNmxEEIIuYMwpbi4OLi4uCA2NhbOzs5yh0P6mjULmDRJ+n3ePGDMGHnj0dPjmKQskwnPXI4jEpuowogdwTrb8zTxdc+xoXLmODf8MCOigsSQz2+W3FD+N3fuq8Rm9uwCk9gAxvlm/LYNlQviYHNERIZgckP526FDwLhx0u8zZ776vQDJ62TC3Ho9ERHlNSY3lL+1bi2NOFy+PDBxotzR5Avm2uuJiCivMLmh/EkIaZ4oKytprigLC7kjyjeM2VCZiMgcsCs45T8rVwJ9+gDp6dJzJjYa2IWXiCh7LLmh/GXNGmDECOn39u2BHj3kjSefYhdeIqKsMbmh/GPdOmDoUOn3MWOA7t3ljSefY68nIiLdWC1F+cO33wKDB0u/jx4tdf9mdRQREeUCkxuS3+bNwKBB0u+ffgosWMDEhoiIco3JDcnr6VNg2DCpd9TIkcDixUxsiIjorbDNDcmrRAlg3z7g2DFg4UImNkRE9NaY3JA8kpMBW1vp99atpQcREVEeYLUUmd4PPwB+fsDdu3JHQkREZoglN2Rae/YAvXpJA/Rt2CDN8G1kmbNgxyWnwtnOBu4O7EKd3/FvRkRvI1+U3KxcuRLe3t6wtbVFvXr1cOnSpSy3Xb9+PRo3bgw3Nze4ubkhMDAw2+0pH/nxR6BnTymx6dsXmDPH6Kd8HJOEETuC0WLRr+iy6je0WPgrRu4IxuOYJKOfm3KHfzMieluyJze7du3C6NGjMWXKFFy9ehU1atRAUFAQnj17pnP7M2fOoGfPnjh9+jQuXrwILy8vtGrVCv/995+JIyeDHDggDcqXlgb07i2Na2Np3JdfbKIKY/dex7k35mA6ey8K4/ZeR2yiyqjnJ8Pxb0ZEeUH25GbRokUYNGgQ+vfvjypVqmDNmjWwt7fHxo0bdW7//fffY9iwYahZsyb8/PywYcMGZGRk4OTJkyaOnPT288/A++9LiU2vXtJEmFZWRj9tVIJK60My09l7UYhK4AdlfsO/GRHlBVmTG5VKhStXriAwMFC9zNLSEoGBgbh48aJex0hMTERqaiqKFCmic31KSgri4uI0HmRCGRnA7NlAairwwQfAli0mSWwAIC45Ndv18TmsJ9Pj34yI8oKsyU1UVBTS09NRvHhxjeXFixfH06dP9TrG2LFjUbJkSY0E6XWzZ8+Gi4uL+uHl5fXWcZMBLC2BQ4eASZOArVsBa9O1YXe2tcl2vVMO6wuj2EQVQp8lIDg8GqGRCSavBuLfjIjyQoHuLTVnzhzs3LkTZ86cgW3mmClvGD9+PEaPHq1+HhcXxwTHFP77DyhVSvrdzQ2YMcPkIbg7KtDE1x1ndVRzNPF1h7sje9+87nFMklZ7lya+7pjT1R8lXe1MEgP/ZkSUF2QtuXF3d4eVlRUiIiI0lkdERKBEiRLZ7rtgwQLMmTMHx44dg7+/f5bbKZVKODs7azzIyE6cAHx9geXLZQ3DxV6BOV390cTXXWN5E193zO3qz67Fr8kvDXn5NyOivCBryY1CoUBAQABOnjyJzp07A4C6cfCIESOy3G/evHmYNWsWjh49ijp16pgoWtLLqVNAx45AUpKU5AwfbvReUdkp6WqH5T1rISpBhfjkVDjZ2sDdkWOmvEmfhrymumf8mxHR25K9Wmr06NHo27cv6tSpg7p162LJkiV4+fIl+vfvDwDo06cPSpUqhdmzZwMA5s6di8mTJ2P79u3w9vZWt81xdHSEo6OjbNdBAH79FWjfXkps2rWTRiKWMbHJ5GLPD8ac5LeGvPybEdHbkD256dGjByIjIzF58mQ8ffoUNWvWxJEjR9SNjMPDw2H52gfk6tWroVKp0K1bN43jTJkyBVOnTjVl6PS6c+eAtm2lxKZNG2DvXkCplDsq0hMb8hKRObEQQgi5gzCluLg4uLi4IDY2lu1v8sqFC0BQEPDypfRz//5Xk2JSgRCbqMLIHcFZNuRd3rMWS1KISFaGfH7LX2dABd+FC1JiExgoTbHAxKbAYUNeIjInLLmhvPH990CXLoC9vdyR0FvInLCSDXmJKL8x5PNb9jY3VEDduAGUKwdkNuL+8EN546E8wYa8RGQOmNyQ4a5ckaqgqlUDDh8GnJzkjojyWGYJTlxyKpztbODuwKSHiAoOJjdkmOBgoGVLICZGem5hIWs4lPfyw0jFRERvgw2KSX9//SWV2ERHAw0aSKU2HFvIrOSXkYqJiN4GS25IP9evAy1aAC9eAO++C/zyC6ujzFBMYir6NfBGz7plYGtjhavh0dh4PgyJqnSTj1RMRJRbTG4oZ3//LSU2z58DdesCR44A7Glmdh7HJGHS/hs4F/JcvayhT1Es61kLo3YEI1GVbvKRiomIcoPVUpSzjAzpZ506wNGjgIuLvPFQnlNXR72W2ADAhZDn2HQhDAMalQPAkYqJqGBgyQ3lzN9fmjfK0xNwdTXaadhDRz7ZTZx5IeQ5BjQshya+7nB35N+DiPI/Jjek2507QGQk0KiR9LxKFaOejj105JXTxJkAOFKxnpikE8mPyQ1pu3cPaNYMiI0Fjh+XekYZUU49dDivkfHlNHFmmSL28GSSmSMm6UT5A9vckKaQECmxefIEKF8e8PU1+imzqxLJ7KFDxuXuqNCaVypTE193KKwsERwejdDIBHYHzwK70RPlHyy5oVfu35cSm//+k6qhTp4EihUz+mlzqhJhDx3jy5w4c9ze6xozgzf2dcewZj5os+wcElXpAFgSkRV9knSWQBKZBpMbkjx4ICU2//4L+PkBp04BHh4mOXVOVSLsoWMaJV3tsLxnLfXEmQ5Ka1x+GI0Bm/9UJzYAqwuzwiSdKP9gtRQBjx8D770HhIcDFStKiU3x4iY7fU5VIuyhYzou9gpU8HBEzTJusLSwwPh9NzQSm0ysLtTGJJ0o/2ByQ0DRokCNGlL7mtOnpS7fJpRZJfJmgtPE1509dGTEkgjDMEknyj9YLUWAUgns3i3NGWXCEpvXvVkl4mRrA3dHdqGVE0siDJNVuyUm6USmx+SmsPrvP2DLFmD8eGlmb4VCtsQmk4s9k5n8JLMk4qyORrIsidCNSTpR/sDkpjB68gRo3hy4exdITQWmTJE7IsqHWBKRO0zSieTH5KawefpU6hV19y5QpgzQt6/cEVE+xpIIIiqImNwUJhERUonNnTuAlxdw5gzg7S13VJTPsSSCiAoa9pYqLJ49A1q0AG7dAkqXlnpFlSsnd1RERER5jslNYZCWBgQFATdvAiVLSolNhQrZ7hKbqELoswQOuW+g/HrfHj58CB/figgPD5c7FCIio2O1VGFgbQ2MGweMGQOcOAH4+GS7OSf/y538fN+mz5yF+/fvY/rMWdiwbq2ssRARGZuFEELIHYQpxcXFwcXFBbGxsXB2dpY7HADSt/2oBBXiklPhbGcDdwcjtXFISgLssv+QjU1UYcSOYJ1z5DTxdeeQ+1nIz/cts9TGrd2XiDm8ECH37qJMmTKyxEJElFuGfH6zWkpmj2OSMGJHMFos+hVdVv2GFgt/xcgdwXgck/R2B46OBnr0kOaKypRDYgNwhu7cys/37eupM+BQqQHsKzWAfcX6+HrqDNliISIyBSY3MopNVGlVYwCvJibMdXuNmBigVSvghx+A7t0BAwrnOOR+7uTX+3bp+h1s2/YdHOr1AAA41OuBbVu/w5837soSDxGRKTC5kZFRvu3HxkqNhy9fBtzdgbVrpRGI9cQh93MnP9632EQVPhw5Dna+DWDj7gUAsHH3gq1vffQaOS7fNHYmIsprTG5klOff9uPigNatgUuXgCJFpMbD1asbdAh9J//Lr72C5JIfJ03861YIQi4cgmuDDzSWuzb4AKEXDuH67VCTx0REZApMbmSUp9/24+OBNm2A338H3NyAkyelmb4NpM8M3UZrJ1SA5ceZzRctnAf7iq9KbTLZuHvBzrc+Fi+cZ/KYiIhMgb2lZBSbqMLIHcFZTkxoUA+b/v2BzZsBV1cpsald+61j0zXkfn7uFZQfZHXfTC2zh5RHn6VayQ0ApEY9wrOtnyGUPaeIqIBgb6kCIk+/7X/zDfDuu8Dx42+d2GTGVsHDETXLuKGCh6M6lvzcKyg/yOq+mdr0mbPgWEm71CaTjbsXHCvWx/SZs0wcGRGR8bHkJh/I9bd9ITQbC7/53AiCw6PRZdVvWa7fP6wBapZxM2oMlL2cSm0ypUY9QuTWzzjuDREVCCy5KQBeb5Ab9VIFd0eFYd/2k5KkNjbbtr1aZuTEBsifvYJI0/SZs+Dol3WpTSYbdy84VGLpDRGZHyY3MnjrBrnJyUDnzsDRo8CIEcCLF0aN93X5sVcQvfLw4UN8t2UL7Ov20Gt7+7o98N2WLZxziojMCpMbE3vrgfuSk4EuXYBjxwAHB+DgQanbt4Ex5LYbd37sFUSvTJ85C3Zl/WGhsENaXFSODwuFHWzLVGfpDRGZFU6caWL6NMjNMkFISQG6dgWOHAHs7YHDh4FGjQw6f15M7ljS1Q7Le9bKF72CSNOJEycR/+A+4kP6GbTf8TTTlf4RERkbkxsTy/XAfSoV8P77UkJjZyeV2DRpYtC5cyo1MqQbt4s9k5n86GEYB+YjImK1lInlukHu998DP/8M2NpKP5s1M/jc7MZNRESFAUtuTCyzQW5WA/dl2SC3Xz/g7l0pqWnRIlfnzqnU6GVKqrpbelxyKpztbODuwBIaIiIqWJjcmFhmg9xxe69rJDg6G+SmpQHp6YBSKXXznj37rc6dXamRvcIKznYKrdGHDW2PQ0REJDcO4ieTHAfuS0sDPvwQePkS2LNHqo7Kg3NmNd3D/G7++OnaY5wLyZ/TKrBEiYiocDPk85slNzLJtkFuWhrQpw/www+AjQ1w9SrQoEGenDOz1Ojyw2gMaFQOtbxcYWEBeLrYYcye6zr3y7EXl5HlRQ8vIiIqPNigOL9JT5fa1+zYAVhbA7t350likymzG/fhUY3xV3g0Pt5yGQM2X0ZY1Mts98uyF5eRvfW4QEREVOgwuclP0tOBAQOknlHW1lLJTadORjnV1wf+xrmQ5+rnSuvsXwpyTavAHl5ERGQoVkvlFxkZwMCBwHffAVZWwM6d0kjERqArYQh+FIOGPkVx4bWEJ1PjHKZVMGZ7mFyPC0SUC2zbRWQemNzkF/fuSQ2HraykKqmuXY12Kl0Jw8bzYVjWsxYAaCQ4DX2KYkanaln+gzd2exhO1EmmwrZdROaD1VL5RaVK0nxR27dLIxEbka6EIVGVjlE7glGrjBt+HtkQqz6sjW/71kF7/5Jws9edQJiiPQwn6iRTYNsuIvPC5EZOQgD/m405NlGF0ArVEfxuS4MnszTUmwmDvcIKI5r7YHnPWqjl5QpVmsA/T+Kw81I43qtYLMtSG1O0h+FEnWQKbNtFZF5YLSUXIYCRI4EdOxC572eMDrHKsjg8r9sBvNklfFnPWth0IQwrToWot2ns647ZXarDM5vi+OgcErC8ag/DiTrJ2Ni2i8i8MLmRgxDAZ58BK1dCWFhgz9ZjOOf+jsYmlx9G49e7kahdxhX/RifBwsICV8OjsfF8GOqUdXvrdgCZCUNMYiom7b+h1ZD43L0oTPjxRpaD98UmqqBKy8j2HHnZHoYTdZIxsW0XkXlhtZSpCQF88QWwbBkAIHLRCsx9I7GxV1hhWc9aOHj9MYKWnPvfWDR/IjhcKmW5/DA6T9oBuNgrkJYhNLqEvy674vioBBV+u/8cDX2K6lyfUw8rovyEbbuIzAuTG1MSAvjqK2DxYun5+vX47/96am02oFE5bLoQplWaciHkOTZdCMOARuXyrB1Abovj45JTsfF8GPo3LKeV4DT0KYppHauypIUKDLbtIjIvrJYyFSGA8eOBBQuk52vWAAMHokjUS3zbtw5S0jJga2OFq+HRqF3GDRvPh2FEcx/U8nLVWLfxfBgGNCwHIG/aAeS2ON7Z1kbdw2pAo3IY0LAcUtIyoLS2RPCjmLeOi8jU2LaLyHwwuTEVlQr44w/p9xUrgMGD8TgmCZP2/62erNJeYYVJ7SrD01mJH4c1wMyD/2g08m3oUxTLetaCk601RjT3gYudzVs3Ns4sjtc1mWZ2xfGv7/d6jJn7DWpUTu8YiPILtu0iMg+cFdyUXr4EjhwBunZFbKIKI3YEq3tIZbaz2XQhDLXKuCE4PFrnaMENfYpiSocquPFvLKqWcsF/bzQ2DijrhmkdqwIAiuqZ6DyOScK4vdc1EpzM4vjsekvldj8iIiJDGfL5zeTG2H79FWjaVGvxvYh4dFp5QT0zdxEHBRYdu4NzIc/xbd86+HjLZZ2Hs1dY4eeRjTD1jbmhGvoURf+G5TBqRzACyrghwNsNVx9G692rKrMEyNDi+NzuR0REZAgmN9kwaXIzcybw9dfA5MnAtGnqxU9ikvA8QQULS4H0DMDCwgJJqnS8TEnDlfBoVC/lgsFbr+g85IjmPvgrPFpnD6eGPkVRq4wbVpwKwZ4h9dFtzUU08XXPsjs3ERFRQWHI53e+6C21cuVKeHt7w9bWFvXq1cOlS5ey3X737t3w8/ODra0tqlevjsOHD5soUgPMni0lNgDg4KBeHBmXjJS0dCSkpOFlSgZepqTh6M2n6LfpEvr/r7t3uaIOsFdY6TxsLS/XLLtuXwh5jlpergCAtAwpZ+XoqkREVNjIntzs2rULo0ePxpQpU3D16lXUqFEDQUFBePbsmc7tf/vtN/Ts2RMff/wxgoOD0blzZ3Tu3Bl///23iSPPxrx5wIQJ0u/ffIPYEZ/hfmQC7kcmIDIhBZP2/40P1v+O7msvouf6P/BXeAy2D3oX7o4KXAh5jhkHb2JSu8q5OnXK/wbWez054uiqRERUmMie3CxatAiDBg1C//79UaVKFaxZswb29vbYuHGjzu2XLl2K1q1bY8yYMahcuTJmzJiB2rVrY8WKFSaOPAsLFwJjx0q/z5iBx0M/w4gdwdgX/B9+v/8csw/fwvk3RwMOicKCo7exsd87sFdY4VzIc1Qv7aI1fkxjn6Io4Wyb7emV1pZa+3F0VSIiKkxkTW5UKhWuXLmCwMBA9TJLS0sEBgbi4sWLOve5ePGixvYAEBQUlOX2JrVkCfDll9LvU6cidvRX6pmGa3m5orizbZZVSudDniM+OQ0D/teF+t/oJNQq44bvB9bDzk/excZ+72BMaz8cvxWBRlmMCtzQpygi4pIxopkvTt+RSr44uioRERU2so5zExUVhfT0dBQvXlxjefHixXH79m2d+zx9+lTn9k+fPtW5fUpKClJSUtTPY2NjAUgNk/JcZtvsr74CPv8cYU+i8Ovf0qzf0THSeTNSErPc/UnkC1R0s0JGSiJcrFJR0c0Kp288xNbfH6BGaVf4l3bF1t8fYF63GkhNTsTF0FeJUv3yRfBV8zIIfhiN59FpWHXsBt71csOklt6wSEtGXFxy3l8vERGRiWR+buvVD0rI6L///hMAxG+//aaxfMyYMaJu3bo697GxsRHbt2/XWLZy5Urh4eGhc/spU6YIAHzwwQcffPDBhxk8Hj16lGN+IWvJjbu7O6ysrBAREaGxPCIiAiVKlNC5T4kSJQzafvz48Rg9erT6eUZGBl68eIGiRYvCwsLiLa9AU1xcHLy8vPDo0SPTDxCYT/Ge6Mb7ohvvizbeE914X7SZ+z0RQiA+Ph4lS5bMcVtZkxuFQoGAgACcPHkSnTt3BiAlHydPnsSIESN07lO/fn2cPHkSn332mXrZ8ePHUb9+fZ3bK5VKKJVKjWWurq55EX6WnJ2dzfKF9TZ4T3TjfdGN90Ub74luvC/azPmeuLi46LWd7HNLjR49Gn379kWdOnVQt25dLFmyBC9fvkT//v0BAH369EGpUqUwe/ZsAMCnn36Kpk2bYuHChWjXrh127tyJy5cvY926dXJeBhEREeUTsic3PXr0QGRkJCZPnoynT5+iZs2aOHLkiLrRcHh4OCwtX3XqatCgAbZv345JkyZhwoQJ8PX1xf79+1GtWjW5LoGIiIjyEdmTGwAYMWJEltVQZ86c0Vr2/vvv4/333zdyVIZTKpWYMmWKVjVYYcZ7ohvvi268L9p4T3TjfdHGe/JKoZtbioiIiMyb7CMUExEREeUlJjdERERkVpjcEBERkVlhcmOglStXwtvbG7a2tqhXrx4uXbqU7fa7d++Gn58fbG1tUb16dRw+fNhEkZqOIfdk/fr1aNy4Mdzc3ODm5obAwMAc72FBZehrJdPOnTthYWGhHvvJnBh6T2JiYjB8+HB4enpCqVSiYsWKhf49BABLlixBpUqVYGdnBy8vL3z++edITjafKVbOnj2LDh06oGTJkrCwsMD+/ftz3OfMmTOoXbs2lEolfHx8sHnzZqPHaWqG3pd9+/ahZcuWKFasGJydnVG/fn0cPXrUNMHKTZ9pEkiyc+dOoVAoxMaNG8XNmzfFoEGDhKurq4iIiNC5/YULF4SVlZWYN2+e+Oeff8SkSZOEjY2NuHHjhokjNx5D70mvXr3EypUrRXBwsLh165bo16+fcHFxEf/++6+JIzcuQ+9LprCwMFGqVCnRuHFj0alTJ9MEayKG3pOUlBRRp04d0bZtW3H+/HkRFhYmzpw5I65du2biyI3L0Pvy/fffC6VSKb7//nsRFhYmjh49Kjw9PcXnn39u4siN5/Dhw2LixIli3759AoD48ccfs93+/v37wt7eXowePVr8888/Yvny5cLKykocOXLENAGbiKH35dNPPxVz584Vly5dEnfv3hXjx48XNjY24urVq6YJWEZMbgxQt25dMXz4cPXz9PR0UbJkSTF79myd23fv3l20a9dOY1m9evXE4MGDjRqnKRl6T96UlpYmnJycxJYtW4wVoixyc1/S0tJEgwYNxIYNG0Tfvn3NLrkx9J6sXr1alC9fXqhUKlOFKAtD78vw4cNF8+bNNZaNHj1aNGzY0KhxykWfD/GvvvpKVK1aVWNZjx49RFBQkBEjk5c+90WXKlWqiGnTpuV9QPkMq6X0pFKpcOXKFQQGBqqXWVpaIjAwEBcvXtS5z8WLFzW2B4CgoKAsty9ocnNP3pSYmIjU1FQUKVLEWGGaXG7vy/Tp0+Hh4YGPP/7YFGGaVG7uyU8//YT69etj+PDhKF68OKpVq4ZvvvkG6enppgrb6HJzXxo0aIArV66oq67u37+Pw4cPo23btiaJOT8y9/+1eSUjIwPx8fFm9f82K/liEL+CICoqCunp6eqRkzMVL14ct2/f1rnP06dPdW7/9OlTo8VpSrm5J28aO3YsSpYsqfWPqSDLzX05f/48vv32W1y7ds0EEZpebu7J/fv3cerUKXz44Yc4fPgwQkJCMGzYMKSmpmLKlCmmCNvocnNfevXqhaioKDRq1AhCCKSlpWHIkCGYMGGCKULOl7L6XxsXF4ekpCTY2dnJFFn+smDBAiQkJKB79+5yh2J0LLkh2cyZMwc7d+7Ejz/+CFtbW7nDkU18fDx69+6N9evXw93dXe5w8o2MjAx4eHhg3bp1CAgIQI8ePTBx4kSsWbNG7tBkdebMGXzzzTdYtWoVrl69in379uHQoUOYMWOG3KFRPrZ9+3ZMmzYNP/zwAzw8POQOx+hYcqMnd3d3WFlZISIiQmN5REQESpQooXOfEiVKGLR9QZObe5JpwYIFmDNnDk6cOAF/f39jhmlyht6X0NBQPHjwAB06dFAvy8jIAABYW1vjzp07qFChgnGDNrLcvFY8PT1hY2MDKysr9bLKlSvj6dOnUKlUUCgURo3ZFHJzX77++mv07t0bAwcOBABUr14dL1++xCeffIKJEydqzMVXWGT1v9bZ2ZmlNpB6YA4cOBC7d+82q1Ly7BS+d0EuKRQKBAQE4OTJk+plGRkZOHnyJOrXr69zn/r162tsDwDHjx/PcvuCJjf3BADmzZuHGTNm4MiRI6hTp44pQjUpQ++Ln58fbty4gWvXrqkfHTt2RLNmzXDt2jV4eXmZMnyjyM1rpWHDhggJCVEnegBw9+5deHp6mkViA+TuviQmJmolMJkJoCiks+mY+//at7Fjxw70798fO3bsQLt27eQOx3TkbtFckOzcuVMolUqxefNm8c8//4hPPvlEuLq6iqdPnwohhOjdu7cYN26cevsLFy4Ia2trsWDBAnHr1i0xZcoUs+wKbsg9mTNnjlAoFGLPnj3iyZMn6kd8fLxcl2AUht6XN5ljbylD70l4eLhwcnISI0aMEHfu3BEHDx4UHh4eYubMmXJdglEYel+mTJkinJycxI4dO8T9+/fFsWPHRIUKFUT37t3luoQ8Fx8fL4KDg0VwcLAAIBYtWiSCg4PFw4cPhRBCjBs3TvTu3Vu9fWZX8DFjxohbt26JlStXmmVXcEPvy/fffy+sra3FypUrNf7fxsTEyHUJJsPkxkDLly8XZcqUEQqFQtStW1f8/vvv6nVNmzYVffv21dj+hx9+EBUrVhQKhUJUrVpVHDp0yMQRG58h96Rs2bICgNZjypQppg/cyAx9rbzOHJMbIQy/J7/99puoV6+eUCqVonz58mLWrFkiLS3NxFEbnyH3JTU1VUydOlVUqFBB2NraCi8vLzFs2DARHR1t+sCN5PTp0zr/T2Teh759+4qmTZtq7VOzZk2hUChE+fLlxaZNm0wet7EZel+aNm2a7fbmjLOCExERkVlhmxsiIiIyK0xuiIiIyKwwuSEiIiKzwuSGiIiIzAqTGyIiIjIrTG6IiIjIrDC5ISIiIrPC5IaIiIjMCpMbIiIiMitMbojIaJ4+fYqRI0eifPnyUCqV8PLyQocOHTQmOfT29oaFhYXWY86cOQCABw8eaCwvUqQImjZtinPnzukVw5YtW/DOO+/A3t4eTk5OaNq0KQ4ePKi1nRAC69atQ7169eDo6AhXV1fUqVMHS5YsQWJiIgBg6tSp6jisrKzg5eWFTz75BC9evMiDu0VEeYXJDREZxYMHDxAQEIBTp05h/vz5uHHjBo4cOYJmzZph+PDhGttOnz4dT5480XiMHDlSY5sTJ07gyZMnOHv2LEqWLIn27dsjIiIi2xi+/PJLDB48GD169MD169dx6dIlNGrUCJ06dcKKFSs0tu3duzc+++wzdOrUCadPn8a1a9fw9ddf48CBAzh27Jh6u6pVq+LJkycIDw/Hpk2bcOTIEQwdOvQt7xYR5SmZ57YiIjPVpk0bUapUKZGQkKC17vVJHsuWLSsWL16c5XHCwsIEABEcHKxedv36dQFAHDhwIMv9Ll68KACIZcuWaa0bPXq0sLGxEeHh4UIIIXbt2iUAiP3792ttm5GRoZ5FecqUKaJGjRpax3Jzc8syDiIyPZbcEFGee/HiBY4cOYLhw4fDwcFBa72rq2uuj52UlITvvvsOAKBQKLLcbseOHXB0dMTgwYO11n3xxRdITU3F3r17AQDff/89KlWqhE6dOmlta2FhARcXF53nePDgAY4ePZptHERketZyB0BE5ickJARCCPj5+em1/dixYzFp0iSNZb/88gsaN26sft6gQQNYWloiMTERQggEBASgRYsWWR7z7t27qFChgs7Eo2TJknB2dsbdu3cBAPfu3UOlSpX0ivXGjRtwdHREeno6kpOTAQCLFi3Sa18iMg0mN0SU54QQBm0/ZswY9OvXT2NZqVKlNJ7v2rULfn5++Pvvv/HVV19h8+bNsLGxyZM4DIm3UqVK+Omnn5CcnIxt27bh2rVrWu2DiEheTG6IKM/5+vrCwsICt2/f1mt7d3d3+Pj4ZLuNl5cXfH194evri7S0NHTp0gV///03lEqlzu0rVqyI8+fPQ6VSaZXePH78GHFxcahYsaJ6W31jVSgU6ljnzJmDdu3aYdq0aZgxY4Ze+xOR8bHNDRHluSJFiiAoKAgrV67Ey5cvtdbHxMS81fG7desGa2trrFq1KsttPvjgAyQkJGDt2rVa6xYsWAAbGxt07doVANCrVy/cvXsXBw4c0NpWCIHY2NgszzNp0iQsWLAAjx8/zsWVEJExMLkhIqNYuXIl0tPTUbduXezduxf37t3DrVu3sGzZMtSvX19j2/j4eDx9+lTjERcXl+WxLSwsMGrUKMyZM0c9Bs2b6tevj08//RRjxozBwoULERoaitu3b2PSpElYunQpFi5cCC8vLwBA9+7d0aNHD/Ts2RPffPMNLl++jIcPH+LgwYMIDAzE6dOns4ylfv368Pf3xzfffJOLu0RERiFnVy0iMm+PHz8Ww4cPF2XLlhUKhUKUKlVKdOzYUZw+fVq9TdmyZQUArcfgwYOFELq7ggshxMuXL4Wbm5uYO3dutjF8++23IiAgQNja2goHBwfRuHFj8dNPP2ltl56eLlavXi3eeecdYW9vL5ydnUVAQIBYunSpSExMFELo7gouhBA7duwQSqVS3bWciORlIYSBLf+IiIiI8jFWSxEREZFZYXJDREREZoXJDREREZkVJjdERERkVpjcEBERkVlhckNERERmhckNERERmRUmN0RERGRWmNwQERGRWWFyQ0RERGaFyQ0RERGZFSY3REREZFb+H81N+nvGgem/AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "# Sample DataFrame creation (you should have this already)\n",
    "# ncse_correct_compare = pd.read_csv('your_data.csv')  # Example of loading data\n",
    "\n",
    "# Define the specific file names\n",
    "file_names_to_highlight = [\n",
    "    'slug_ar00500_periodical_ewj_issue_ewj_01081858_page_number_5.txt',\n",
    "    'slug_ar00801_periodical_pc_issue_tec_01051889_page_number_8.txt'\n",
    "]\n",
    "\n",
    "# Filter the DataFrame for these specific file names\n",
    "highlight_df = ncse_correct_compare[ncse_correct_compare['File Name'].isin(file_names_to_highlight)]\n",
    "\n",
    "# Create the scatter plot with Seaborn\n",
    "ax = sns.scatterplot(data=ncse_correct_compare, x='CER OCR', y='CER Corrected')\n",
    "\n",
    "# Get the default color used by Seaborn for scatter plots\n",
    "default_color = sns.color_palette()[0]\n",
    "\n",
    "# Set the y-axis limit\n",
    "plt.ylim([0, 1.05])\n",
    "\n",
    "# Add the line x = y\n",
    "plt.plot([0, 1.05], [0, 1.05], color='red', linestyle='--')\n",
    "\n",
    "# Plot the specific points with triangles and black outline using the default color\n",
    "sns.scatterplot(data=highlight_df, x='CER OCR', y='CER Corrected', \n",
    "                marker='^',  # Triangle marker\n",
    "                edgecolor='black',  # Black outline\n",
    "                color=default_color,  # Match the default scatter plot color\n",
    "                s=100,  # Size of the markers\n",
    "              #  label='Highlighted Examples', \n",
    "                ax=ax)  # Ensure it's on the same axes\n",
    "\n",
    "# Optionally, add labels and title\n",
    "plt.xlabel('CER OCR')\n",
    "plt.ylabel('CER Corrected')\n",
    "plt.title('Comparison of Original OCR vs. Corrected OCR\\n for Claude Opus Full Instructions')\n",
    "\n",
    "# Save the figure\n",
    "plt.savefig(os.path.join(save_figs, \"cer_relationship.pdf\"))\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabularx}{\\textwidth}{>{\\raggedright\\arraybackslash}X>{\\raggedright\\arraybackslash}X>{\\raggedright\\arraybackslash}X}\n",
      "\\toprule\n",
      "\\textbf{Ground Truth}  & \\textbf{Original OCR}  & \\textbf{Corrected}\\\\\n",
      "\\midrule\n",
      "‘The Plague and the Printing Press: being a Bibliographical Record of the Literature relative to the Plague, printed in England since the Year 1480,’ with introduction and numerous notes, by Henry R. Plomer, is the attractive title of a book which Mr. Elliot Stock is about to publish by subscription.\n",
      " & \n",
      "\\texttt{* The Plague and the Printing Press : being a Bibliographical Record of theLiterature relative to the Plague , printed in England since attractive numerous the Year notes title , 1480 of by a Henry , ' book with R which introduction . Plomer Mr . , is Elliot and theStock is about to publish by subscription .} & \n",
      "The Plague and the Printing Press: being a Bibliographical Record of the Literature relative to the Plague, printed in England since the Year 1480, by Henry R. Plomer, with introduction and numerous attractive title notes, is a book which Mr. Elliot Stock is about to publish by subscription.  \\\\\n",
      "\\bottomrule\n",
      "\\end{tabularx}\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "from string import Template\n",
    "\n",
    "def escape_latex(text):\n",
    "    \"\"\"Escape special LaTeX characters.\"\"\"\n",
    "    special_chars = {\n",
    "        '&': r'\\&',\n",
    "        '%': r'\\%',\n",
    "        '$': r'\\$',\n",
    "        '#': r'\\#',\n",
    "        '_': r'\\_',\n",
    "        '{': r'\\{',\n",
    "        '}': r'\\}',\n",
    "        '~': r'\\textasciitilde{}',\n",
    "        '^': r'\\textasciicircum{}',\n",
    "        '\\\\': r'\\textbackslash{}',\n",
    "    }\n",
    "    return ''.join(special_chars.get(c, c) for c in text)\n",
    "\n",
    "gt_15091890 = \"The Leadenhall Press will issue Mr. Jerome K. Jerome's new book, 'Told after Supper.' The publishers thought it necessary to bring out a far-thing booklet, in order to secure the copyright.\"\n",
    "raw_ocr_15091890 = \"\"\"Leadenhall ^^ ^ K ^ -J ^ b ^ r « E-fc «« I I Press 1 _ # * A J ^ . « M ^« will -W »»» I I issue < K »•» « M H ^ % Mr !>/¦ «« . Jerome I ^ % *»^ \"V *^»^ rf\\'V T K . Jerome publis \\' h ers new have book though , * Told t it after necessary Supper to . \\'the bring copyri out a ght far . thing booklet , in order to secure*\"\"\"\n",
    "clocrc_opus_full_15091890 = \"The Leadenhall Press will issue Mr. Jerome K. Jerome's new book, 'Told after Supper.' The publishers thought it necessary to bring out a far-thing booklet, in order to secure the copyright.\"\n",
    "\n",
    "\n",
    "gt_tec_01051889 = \"\"\"‘The Plague and the Printing Press: being a Bibliographical Record of the Literature relative to the Plague, printed in England since the Year 1480,’ with introduction and numerous notes, by Henry R. Plomer, is the attractive title of a book which Mr. Elliot Stock is about to publish by subscription.\n",
    "\"\"\"\n",
    "raw_ocr_tec_01051889 = \"\"\"* The Plague and the Printing Press : being a Bibliographical Record of theLiterature relative to the Plague , printed in England since attractive numerous the Year notes title , 1480 of by a Henry , ' book with R which introduction . Plomer Mr . , is Elliot and theStock is about to publish by subscription .\"\"\"\n",
    "clocrc_opus_01051889 = \"\"\"The Plague and the Printing Press: being a Bibliographical Record of the Literature relative to the Plague, printed in England since the Year 1480, by Henry R. Plomer, with introduction and numerous attractive title notes, is a book which Mr. Elliot Stock is about to publish by subscription.\"\"\"\n",
    "\n",
    "# Your text strings (gt_15091890, clocrc_opus_full_15091890, raw_ocr_15091890)\n",
    "\n",
    "# Generate table content\n",
    "# Generate table content\n",
    "table_content = f\"\"\"\\\\begin{{tabularx}}{{\\\\textwidth}}{{>{{\\\\raggedright\\\\arraybackslash}}X>{{\\\\raggedright\\\\arraybackslash}}X>{{\\\\raggedright\\\\arraybackslash}}X}}\n",
    "\\\\toprule\n",
    "\\\\textbf{{Ground Truth}}  & \\\\textbf{{Original OCR}}  & \\\\textbf{{Corrected}}\\\\\\\\\n",
    "\\\\midrule\n",
    "{escape_latex(gt_tec_01051889)} & \n",
    "\\\\texttt{{{escape_latex(raw_ocr_tec_01051889)}}} & \n",
    "{escape_latex(clocrc_opus_01051889)}  \\\\\\\\\n",
    "\\\\bottomrule\n",
    "\\\\end{{tabularx}}\"\"\"\n",
    "print(table_content)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabularx}{\\textwidth}{>{\\raggedright\\arraybackslash}X>{\\raggedright\\arraybackslash}X>{\\raggedright\\arraybackslash}X}\n",
      "\\toprule\n",
      "\\textbf{Ground Truth} & \\textbf{Original OCR} & \\textbf{Corrected}  \\\\\n",
      "\\midrule\n",
      " and hence ever in the jaws of dependence and credulity  & \n",
      "\\texttt{hence y suffered ever in them the to jaws sink of into dependenceforgetfulness on earth } & \n",
      "and hence they are said to be incapable of other endowments. In lieu of justice, women in all countries and in all ranks, with the most marvellous inconsistency, but instead of recognising expediency, have ever\\\\\n",
      "\\bottomrule\n",
      "\\end{tabularx}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Your text strings (gt_15091890, clocrc_opus_full_15091890, raw_ocr_15091890)\n",
    "\n",
    "gt_ewj_issue_ewj_01081858 = \" and hence ever in the jaws of dependence and credulity\"\n",
    "raw_ocr_ewj_issue_ewj_01081858 = \"\"\"hence y suffered ever in them the to jaws sink of into dependenceforgetfulness on earth \"\"\"\n",
    "clocrc_ewj_issue_ewj_01081858 = \"\"\"and hence they are said to be incapable of other endowments. In lieu of justice, women in all countries and in all ranks, with the most marvellous inconsistency, but instead of recognising expediency, have ever\"\"\"\n",
    "# Generate table content\n",
    "# Generate table content\n",
    "table_content = f\"\"\"\\\\begin{{tabularx}}{{\\\\textwidth}}{{>{{\\\\raggedright\\\\arraybackslash}}X>{{\\\\raggedright\\\\arraybackslash}}X>{{\\\\raggedright\\\\arraybackslash}}X}}\n",
    "\\\\toprule\n",
    "\\\\textbf{{Ground Truth}} & \\\\textbf{{Original OCR}} & \\\\textbf{{Corrected}}  \\\\\\\\\n",
    "\\\\midrule\n",
    "{escape_latex(gt_ewj_issue_ewj_01081858)}  & \n",
    "\\\\texttt{{{escape_latex(raw_ocr_ewj_issue_ewj_01081858)}}} & \n",
    "{escape_latex(clocrc_ewj_issue_ewj_01081858)}\\\\\\\\\n",
    "\\\\bottomrule\n",
    "\\\\end{{tabularx}}\"\"\"\n",
    "print(table_content)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HAIKU Deepdive\n",
    "\n",
    "The reviewer says my results are incorrect. I check this here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>File Name</th>\n",
       "      <th>WER</th>\n",
       "      <th>CER</th>\n",
       "      <th>lev_dist</th>\n",
       "      <th>CER_erp</th>\n",
       "      <th>CER_orig</th>\n",
       "      <th>CER_improve</th>\n",
       "      <th>both_improve</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>slug_ar04900_periodical_ewj_issue_ewj_01051860...</td>\n",
       "      <td>0.138776</td>\n",
       "      <td>0.100140</td>\n",
       "      <td>301</td>\n",
       "      <td>33.796296</td>\n",
       "      <td>0.151261</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>slug_ar00408_periodical_ns_issue_vm2-ncseprodu...</td>\n",
       "      <td>0.753187</td>\n",
       "      <td>0.746250</td>\n",
       "      <td>7132</td>\n",
       "      <td>-943.108504</td>\n",
       "      <td>0.071541</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>slug_ar00502_periodical_t_issue_ttw_16051868_p...</td>\n",
       "      <td>0.941909</td>\n",
       "      <td>0.597917</td>\n",
       "      <td>861</td>\n",
       "      <td>0.577367</td>\n",
       "      <td>0.601389</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>slug_ar00200_periodical_l_issue_cld_30071853_p...</td>\n",
       "      <td>0.784585</td>\n",
       "      <td>0.784473</td>\n",
       "      <td>4656</td>\n",
       "      <td>-973.441109</td>\n",
       "      <td>0.073080</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>slug_ar01400_periodical_ns_issue_ns2_02101852_...</td>\n",
       "      <td>0.824613</td>\n",
       "      <td>0.809397</td>\n",
       "      <td>6111</td>\n",
       "      <td>-1704.142012</td>\n",
       "      <td>0.044863</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>slug_ar00602_periodical_l_issue_vm2-ncseproduc...</td>\n",
       "      <td>0.158940</td>\n",
       "      <td>0.048884</td>\n",
       "      <td>57</td>\n",
       "      <td>73.255814</td>\n",
       "      <td>0.182784</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178</th>\n",
       "      <td>slug_ad01226_periodical_t_issue_ttw_21121867_p...</td>\n",
       "      <td>0.551724</td>\n",
       "      <td>0.434783</td>\n",
       "      <td>181</td>\n",
       "      <td>37.716263</td>\n",
       "      <td>0.698068</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>slug_ar01401_periodical_ns_issue_ns2_02101852_...</td>\n",
       "      <td>0.186275</td>\n",
       "      <td>0.149550</td>\n",
       "      <td>84</td>\n",
       "      <td>-232.000000</td>\n",
       "      <td>0.045045</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>180</th>\n",
       "      <td>slug_ar00500_periodical_pc_issue_tec_15091890_...</td>\n",
       "      <td>0.186441</td>\n",
       "      <td>0.016012</td>\n",
       "      <td>23</td>\n",
       "      <td>54.166667</td>\n",
       "      <td>0.034934</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>slug_ar03200_periodical_mruc_issue_vm2-ncsepro...</td>\n",
       "      <td>0.260492</td>\n",
       "      <td>0.245239</td>\n",
       "      <td>967</td>\n",
       "      <td>-633.076923</td>\n",
       "      <td>0.033453</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>91 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             File Name       WER       CER  \\\n",
       "91   slug_ar04900_periodical_ewj_issue_ewj_01051860...  0.138776  0.100140   \n",
       "92   slug_ar00408_periodical_ns_issue_vm2-ncseprodu...  0.753187  0.746250   \n",
       "93   slug_ar00502_periodical_t_issue_ttw_16051868_p...  0.941909  0.597917   \n",
       "94   slug_ar00200_periodical_l_issue_cld_30071853_p...  0.784585  0.784473   \n",
       "95   slug_ar01400_periodical_ns_issue_ns2_02101852_...  0.824613  0.809397   \n",
       "..                                                 ...       ...       ...   \n",
       "177  slug_ar00602_periodical_l_issue_vm2-ncseproduc...  0.158940  0.048884   \n",
       "178  slug_ad01226_periodical_t_issue_ttw_21121867_p...  0.551724  0.434783   \n",
       "179  slug_ar01401_periodical_ns_issue_ns2_02101852_...  0.186275  0.149550   \n",
       "180  slug_ar00500_periodical_pc_issue_tec_15091890_...  0.186441  0.016012   \n",
       "181  slug_ar03200_periodical_mruc_issue_vm2-ncsepro...  0.260492  0.245239   \n",
       "\n",
       "     lev_dist      CER_erp  CER_orig  CER_improve  both_improve  \n",
       "91        301    33.796296  0.151261            1             1  \n",
       "92       7132  -943.108504  0.071541            0             0  \n",
       "93        861     0.577367  0.601389            1             1  \n",
       "94       4656  -973.441109  0.073080            0             0  \n",
       "95       6111 -1704.142012  0.044863            0             0  \n",
       "..        ...          ...       ...          ...           ...  \n",
       "177        57    73.255814  0.182784            1             1  \n",
       "178       181    37.716263  0.698068            1             1  \n",
       "179        84  -232.000000  0.045045            0             0  \n",
       "180        23    54.166667  0.034934            1             1  \n",
       "181       967  -633.076923  0.033453            0             0  \n",
       "\n",
       "[91 rows x 8 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "HAIKU_comparison = ncse_performance_eval.merge(ncse_error_reduction.drop(columns = ['lev_dist', 'WER']), on = ['File Name', 'type'], suffixes=['', '_erp'])\n",
    "HAIKU_comparison = HAIKU_comparison.merge(ncse_raw_ocr_eval.drop(columns = ['type', 'lev_dist', 'WER']), on = 'File Name', suffixes=['', '_orig'])\n",
    "\n",
    "HAIKU_comparison = HAIKU_comparison.loc[HAIKU_comparison['type']=='full__claude-3-haiku-20240307'].drop(columns = 'type')\n",
    "HAIKU_comparison['CER_improve'] = (HAIKU_comparison['CER']<HAIKU_comparison['CER_orig'])*1\n",
    "\n",
    "HAIKU_comparison['both_improve'] = (HAIKU_comparison['CER_improve'] & (HAIKU_comparison['CER_erp']>0))*1\n",
    "\n",
    "HAIKU_comparison\n",
    "\n",
    "HAIKU_comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CER_improve  both_improve\n",
       "0            0               36\n",
       "1            1               55\n",
       "dtype: int64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "HAIKU_comparison.groupby(['CER_improve', 'both_improve']).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CER</th>\n",
       "      <th>CER_erp</th>\n",
       "      <th>CER_orig</th>\n",
       "      <th>CER_reduce</th>\n",
       "      <th>both_reduce</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>91.000000</td>\n",
       "      <td>91.000000</td>\n",
       "      <td>91.000000</td>\n",
       "      <td>91.000000</td>\n",
       "      <td>91.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.399860</td>\n",
       "      <td>-237.413120</td>\n",
       "      <td>0.327113</td>\n",
       "      <td>0.604396</td>\n",
       "      <td>0.604396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.272223</td>\n",
       "      <td>513.194110</td>\n",
       "      <td>0.298238</td>\n",
       "      <td>0.491689</td>\n",
       "      <td>0.491689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-2113.829787</td>\n",
       "      <td>0.021327</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.132613</td>\n",
       "      <td>-214.243243</td>\n",
       "      <td>0.059740</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.440058</td>\n",
       "      <td>14.037267</td>\n",
       "      <td>0.171853</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.594384</td>\n",
       "      <td>39.534739</td>\n",
       "      <td>0.606182</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.114332</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>1.296296</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             CER      CER_erp   CER_orig  CER_reduce  both_reduce\n",
       "count  91.000000    91.000000  91.000000   91.000000    91.000000\n",
       "mean    0.399860  -237.413120   0.327113    0.604396     0.604396\n",
       "std     0.272223   513.194110   0.298238    0.491689     0.491689\n",
       "min     0.000000 -2113.829787   0.021327    0.000000     0.000000\n",
       "25%     0.132613  -214.243243   0.059740    0.000000     0.000000\n",
       "50%     0.440058    14.037267   0.171853    1.000000     1.000000\n",
       "75%     0.594384    39.534739   0.606182    1.000000     1.000000\n",
       "max     1.114332   100.000000   1.296296    1.000000     1.000000"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "HAIKU_comparison[['CER', 'CER_erp', 'CER_orig', 'CER_reduce', 'both_reduce']].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1.588235294117647"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(0.17-0.44)/0.17"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1740640/1550825786.py:5: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  HAIKU_comparison.groupby('group')[['CER_erp', 'CER', 'CER_orig']].median()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CER_erp</th>\n",
       "      <th>CER</th>\n",
       "      <th>CER_orig</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>group</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Low</th>\n",
       "      <td>-400.000000</td>\n",
       "      <td>0.177686</td>\n",
       "      <td>0.047359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>med</th>\n",
       "      <td>10.460251</td>\n",
       "      <td>0.308537</td>\n",
       "      <td>0.171853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>High</th>\n",
       "      <td>21.470588</td>\n",
       "      <td>0.488812</td>\n",
       "      <td>0.647160</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          CER_erp       CER  CER_orig\n",
       "group                                \n",
       "Low   -400.000000  0.177686  0.047359\n",
       "med     10.460251  0.308537  0.171853\n",
       "High    21.470588  0.488812  0.647160"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create three groups\n",
    "HAIKU_comparison['group'] = pd.qcut(HAIKU_comparison['CER_orig'], q=3, labels=['Low', 'med', 'High'])\n",
    "\n",
    "# Calculate median CER_erp for each group\n",
    "HAIKU_comparison.groupby('group')[['CER_erp', 'CER', 'CER_orig']].median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data_strat = ncse_performance_eval.merge(ncse_error_reduction.drop(columns = ['lev_dist', 'WER']), on = ['File Name', 'type'], suffixes=['', '_erp'])\n",
    "all_data_strat = all_data_strat.merge(ncse_raw_ocr_eval.drop(columns = ['type', 'lev_dist', 'WER']), on = 'File Name', suffixes=['', '_orig'])\n",
    "\n",
    "all_data_strat = all_data_strat.loc[all_data_strat['type']=='full__claude-3-opus-20240229']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1740640/3767867007.py:5: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  all_data_strat.groupby('group')[['CER_erp', 'CER', 'CER_orig']].median().round(2).to_csv('data/stratified_opus.csv')\n"
     ]
    }
   ],
   "source": [
    "# Create three groups\n",
    "all_data_strat['group'] = pd.qcut(all_data_strat['CER_orig'], q=3, labels=['Low', 'med', 'High'])\n",
    "\n",
    "# all_data_strat median CER_erp for each group\n",
    "all_data_strat.groupby('group')[['CER_erp', 'CER', 'CER_orig']].median().round(2).to_csv('data/stratified_opus.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}