bootstraping from scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3sNKZq4XrXQh"
   },
   "source": [
    "# <font color='red'><b>Bootstrap from scratch</b> </font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1343,
   "metadata": {
    "id": "m6ag91ijrQOs"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import load_boston \n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1344,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "CcHOsONTt1K_",
    "outputId": "2ddc6415-086b-4204-d68a-e2159aa7a430"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function load_boston is deprecated; `load_boston` is deprecated in 1.0 and will be removed in 1.2.\n",
      "\n",
      "    The Boston housing prices dataset has an ethical problem. You can refer to\n",
      "    the documentation of this function for further details.\n",
      "\n",
      "    The scikit-learn maintainers therefore strongly discourage the use of this\n",
      "    dataset unless the purpose of the code is to study and educate about\n",
      "    ethical issues in data science and machine learning.\n",
      "\n",
      "    In this special case, you can fetch the dataset from the original\n",
      "    source::\n",
      "\n",
      "        import pandas as pd\n",
      "        import numpy as np\n",
      "\n",
      "\n",
      "        data_url = \"http://lib.stat.cmu.edu/datasets/boston\"\n",
      "        raw_df = pd.read_csv(data_url, sep=\"\\s+\", skiprows=22, header=None)\n",
      "        data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]])\n",
      "        target = raw_df.values[1::2, 2]\n",
      "\n",
      "    Alternative datasets include the California housing dataset (i.e.\n",
      "    :func:`~sklearn.datasets.fetch_california_housing`) and the Ames housing\n",
      "    dataset. You can load the datasets as follows::\n",
      "\n",
      "        from sklearn.datasets import fetch_california_housing\n",
      "        housing = fetch_california_housing()\n",
      "\n",
      "    for the California housing dataset and::\n",
      "\n",
      "        from sklearn.datasets import fetch_openml\n",
      "        housing = fetch_openml(name=\"house_prices\", as_frame=True)\n",
      "\n",
      "    for the Ames housing dataset.\n",
      "    \n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "boston = load_boston()\n",
    "x=boston.data #independent variables\n",
    "y=boston.target #target variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1345,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pc1htEFYuLRj",
    "outputId": "be1f9d3e-0091-4016-fb4a-0bf8e7b387a6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(506, 13)"
      ]
     },
     "execution_count": 1345,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1346,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kQle3T_wuOa3",
    "outputId": "fdd138bd-afe6-47e4-f6e3-79dd0952d10c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[6.3200e-03, 1.8000e+01, 2.3100e+00, 0.0000e+00, 5.3800e-01,\n",
       "        6.5750e+00, 6.5200e+01, 4.0900e+00, 1.0000e+00, 2.9600e+02,\n",
       "        1.5300e+01, 3.9690e+02, 4.9800e+00],\n",
       "       [2.7310e-02, 0.0000e+00, 7.0700e+00, 0.0000e+00, 4.6900e-01,\n",
       "        6.4210e+00, 7.8900e+01, 4.9671e+00, 2.0000e+00, 2.4200e+02,\n",
       "        1.7800e+01, 3.9690e+02, 9.1400e+00],\n",
       "       [2.7290e-02, 0.0000e+00, 7.0700e+00, 0.0000e+00, 4.6900e-01,\n",
       "        7.1850e+00, 6.1100e+01, 4.9671e+00, 2.0000e+00, 2.4200e+02,\n",
       "        1.7800e+01, 3.9283e+02, 4.0300e+00],\n",
       "       [3.2370e-02, 0.0000e+00, 2.1800e+00, 0.0000e+00, 4.5800e-01,\n",
       "        6.9980e+00, 4.5800e+01, 6.0622e+00, 3.0000e+00, 2.2200e+02,\n",
       "        1.8700e+01, 3.9463e+02, 2.9400e+00],\n",
       "       [6.9050e-02, 0.0000e+00, 2.1800e+00, 0.0000e+00, 4.5800e-01,\n",
       "        7.1470e+00, 5.4200e+01, 6.0622e+00, 3.0000e+00, 2.2200e+02,\n",
       "        1.8700e+01, 3.9690e+02, 5.3300e+00]])"
      ]
     },
     "execution_count": 1346,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AEa_HqRZloH4"
   },
   "source": [
    "## <font color='red'><b>Task 1</b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YQ5q8IxHNRk3"
   },
   "source": [
    "<font color='red'> <b>Step - 1</b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GJCFCaOzl7Mr"
   },
   "source": [
    "*  <font color='blue'><b>Creating samples</b></font><br>\n",
    "    <b> Randomly create 30 samples from the whole boston data points</b>\n",
    "    *  Creating each sample: Consider any random 303(60% of 506) data points from whole data set and then replicate any 203 points from the sampled points\n",
    "    \n",
    "     For better understanding of this procedure lets check this examples, assume we have 10 data points [1,2,3,4,5,6,7,8,9,10], first we take 6 data points randomly , consider we have selected [4, 5, 7, 8, 9, 3] now we will replicate 4 points from [4, 5, 7, 8, 9, 3], consder they are [5, 8, 3,7] so our final sample will be [4, 5, 7, 8, 9, 3, 5, 8, 3,7]\n",
    "* <font color='blue'><b> Create 30 samples </b></font>\n",
    "    *  Note that as a part of the Bagging when you are taking the random samples <b>make sure each of the sample will have different set of columns</b><br>\n",
    "Ex: Assume we have 10 columns[1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10] for the first sample we will select [3, 4, 5, 9, 1, 2] and for the second sample  [7, 9, 1, 4, 5, 6, 2] and so on...\n",
    "Make sure each sample will have atleast 3 feautres/columns/attributes\n",
    "\n",
    "* <font color='red'><b> Note - While selecting the random 60% datapoints from the whole data, make sure that the selected datapoints are all exclusive, repetition is not allowed. </b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zUqFEBSvNjCa"
   },
   "source": [
    "<font color='red'><b>Step - 2 </b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uqi9AhCYNq3Z"
   },
   "source": [
    "<font color='blue'><b>Building High Variance Models on each of the sample and finding train MSE value</b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-lLBnZHXOFln"
   },
   "source": [
    "*  Build a regression trees on each of 30 samples.\n",
    "*  Computed the predicted values of each data point(506 data points) in your corpus.\n",
    "*  Predicted house price of $i^{th}$ data point $y^{i}_{pred} =  \\frac{1}{30}\\sum_{k=1}^{30}(\\text{predicted value of } x^{i} \\text{ with } k^{th} \\text{ model})$\n",
    "*  Now calculate the $MSE =  \\frac{1}{506}\\sum_{i=1}^{506}(y^{i} - y^{i}_{pred})^{2}$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Kls23JLnSN23"
   },
   "source": [
    "<font color='red'> <b>Step - 3 </b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rz2GchkGSWnh"
   },
   "source": [
    "*  <font color='blue'><b>Calculating the OOB score </b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DGHkVV2kSibm"
   },
   "source": [
    "*  Predicted house price of $i^{th}$ data point $y^{i}_{pred} =  \\frac{1}{k}\\sum_{\\text{k= model which was buit on samples not included } x^{i}}(\\text{predicted value of } x^{i} \\text{ with } k^{th} \\text{ model})$.\n",
    "*  Now calculate the $OOB Score =  \\frac{1}{506}\\sum_{i=1}^{506}(y^{i} - y^{i}_{pred})^{2}$."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RK860ocxTyoz"
   },
   "source": [
    "# <font color='red'><b>Task 2</b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1dme-N6TUCrY"
   },
   "source": [
    "*  <font color='blue'><b>Computing CI of OOB Score and Train MSE</b></font>\n",
    "  *   Repeat Task 1 for 35 times, and for each iteration store the Train MSE and OOB score </li>\n",
    "<li> After this we will have 35 Train MSE values and 35 OOB scores </li>\n",
    "<li> using these 35 values (assume like a sample) find the confidence intravels of MSE and OOB Score </li>\n",
    "<li> report CI of MSE and CI of OOB Score </li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "e0yGBuryOwHz"
   },
   "source": [
    "## <font color='red'><b>Task 1</b></font>\n",
    "\n",
    "<font color='blue'><b>Step - 1</b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "f_oWoN97BhDY"
   },
   "source": [
    "*  <font color='blue'><b> code for generating samples</b></font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1347,
   "metadata": {
    "id": "j-eTJhQWfpwP"
   },
   "outputs": [],
   "source": [
    "def generating_samples(input_data,target_data):\n",
    "  #selecting_rows=np.random.randint(0,x.shape[0],303)\n",
    "  selecting_rows=np.random.choice(list(range(len(x))),303,replace=False)\n",
    "  #replacing_rows=np.random.randint(0,selecting_rows.shape[0],203)\n",
    "  replacing_rows=np.random.choice(list(range(len(selecting_rows))),203,replace=False)\n",
    "  selecting_columns=np.random.randint(3,13,np.random.randint(3,13))\n",
    "  sample_data=(input_data[selecting_rows[:,None],selecting_columns])\n",
    "  target_of_sample_data=(target_data[selecting_rows])\n",
    "  replicated_sample_data=list(sample_data[replacing_rows])\n",
    "  target_of_replicated_sample_data=(target_of_sample_data[replacing_rows])\n",
    "  final_sample_data=np.vstack([sample_data,replicated_sample_data])\n",
    "  final_target_data=np.vstack([target_of_sample_data.reshape(-1,1),target_of_replicated_sample_data.reshape(-1,1)])\n",
    "  return list(final_sample_data),list(final_target_data),list(selecting_rows),list(selecting_columns)\n",
    "  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "b4LSsmn4Jn2_"
   },
   "source": [
    "*  <font color='blue'> <b>Create 30 samples </b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1349,
   "metadata": {
    "id": "XXlKWjCcBvTk"
   },
   "outputs": [],
   "source": [
    "# Use generating_samples function to create 30 samples \n",
    "# store these created samples in a list\n",
    "list_input_data =[]\n",
    "list_output_data =[]\n",
    "list_selected_row= []\n",
    "list_selected_columns=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1350,
   "metadata": {
    "id": "xU7nbN52tvLF"
   },
   "outputs": [],
   "source": [
    "for i in range(0,30):\n",
    "  a,b,c,d=generating_samples(x,y)\n",
    "  list_input_data.append(a)\n",
    "  list_output_data.append(b)\n",
    "  list_selected_row.append(c)\n",
    "  list_selected_columns.append(d)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7Pv-mkZkO6dh"
   },
   "source": [
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "whaHCPB0O8qF"
   },
   "source": [
    "<font color='red'><b>Step - 2 </b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XBy4zXSWPtU8"
   },
   "source": [
    "<font color='orange'><b>Flowchart for building tree</b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5xvH06HPQBdP"
   },
   "source": [
    "![alt text](https://i.imgur.com/pcXfSmp.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WRwPO_uHQjul"
   },
   "source": [
    "*  <font color='blue'><b> code for building regression trees</b></font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1352,
   "metadata": {
    "id": "YWQp6tRwMthq"
   },
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeRegressor\n",
    "list_of_all_models=[]\n",
    "for i in range(len(list_input_data)):\n",
    "  model=DecisionTreeRegressor()\n",
    "  model.fit(list_input_data[i],list_output_data[i])\n",
    "  list_of_all_models.append(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "21j8BKfAQ1U8"
   },
   "source": [
    "<font color='orange'><b>Flowchart for calculating MSE </b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8Q0mTBD2RBx_"
   },
   "source": [
    "![alt text](https://i.imgur.com/sPEE618.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6e-UamlHRjPy"
   },
   "source": [
    "After getting predicted_y for each data point, we can use sklearns mean_squared_error to calculate the MSE between predicted_y and actual_y."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TnIMT7_oR312"
   },
   "source": [
    "*  <font color='blue'><b> Write code for calculating MSE</b></font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1353,
   "metadata": {
    "id": "tSVgrRdyvodj"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1354,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "qWhcvMRWRA9b",
    "outputId": "5f8c5069-5c7b-499e-8f30-47a6b5656371"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.19131052371541513"
      ]
     },
     "execution_count": 1354,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted_values=[]\n",
    "for i in range(len(list_of_all_models)):\n",
    "  predicted_values.append(list_of_all_models[i].predict(x[:,list_selected_columns[i]]))\n",
    "predict=np.median(np.array(predicted_values),axis=0)\n",
    "#mean squared error\n",
    "mean_squared_error(y,predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RuclPDMnSz8F"
   },
   "source": [
    "<font color='blue'><b>Step - 3 </b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ESb9FSIDTM5V"
   },
   "source": [
    "<font color='orange'><b>Flowchart for calculating OOB score</b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HB-d6NMETbd9"
   },
   "source": [
    "![alt text](https://i.imgur.com/95S5Mtm.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WW3GOcFzTqbt"
   },
   "source": [
    "Now calculate the $OOB Score =  \\frac{1}{506}\\sum_{i=1}^{506}(y^{i} - y^{i}_{pred})^{2}$."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zBqcS03pUYSZ"
   },
   "source": [
    "*  <font color='blue'><b> code for calculating OOB score </b></font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1356,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "zjULApFSn0Vf",
    "outputId": "246a3d98-59f4-4d7c-c442-b9f3d7312d3b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "oob_score is :  14.296114408167083\n"
     ]
    }
   ],
   "source": [
    "oob_score=[]\n",
    "for i in range(len(x)):\n",
    "    predict_oob=[]\n",
    "    for j in range(len(list_of_all_models)):\n",
    "        if i not in list_selected_row[j]:\n",
    "            predict_oob.append(list_of_all_models[j].predict(x[i][list_selected_columns[j]].reshape(1,-1)))\n",
    "    oob_score.append(np.median(predict_oob))\n",
    "print(\"oob_score is : \",(mean_squared_error(y,oob_score)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sbuiwX3OUjUI"
   },
   "source": [
    "# <font color='blue'><b>Task 2</b></font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1357,
   "metadata": {
    "id": "g_8g0S23W4t3"
   },
   "outputs": [],
   "source": [
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1358,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 49,
     "referenced_widgets": [
      "fef1d524c5884f69b953d1fc288b46b2",
      "bff3c595a11044deb8f21bc6e8e6086d",
      "165ea5d2cd6443739e8d004bb13c9bea",
      "9284e14a76954cb48d3ef795f44f0e3f",
      "32d833a3473a4b66914702bd214dc121",
      "8bc496db1ff24bb2a47df4c988de1268",
      "f12ec296ae444149a1b5bf88e271c20e",
      "72396c4af831487fb3cbc01be369a06c",
      "0249e86337c340afb12485dc1471647c",
      "4b5257a8e2594d8b8fe977d726c06bf5",
      "0c21d0cbb6f044afa756e9c7f08f78e4"
     ]
    },
    "id": "ceW5-D88Uswi",
    "outputId": "6ab5920b-7285-4cba-c4b5-2d0c652924de"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fef1d524c5884f69b953d1fc288b46b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/35 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "obb_score_full=[]\n",
    "mse_full=[]\n",
    "for i in tqdm(range(35)):\n",
    "  list_input_data =[]\n",
    "  list_output_data =[]\n",
    "  list_selected_row= []\n",
    "  list_selected_columns=[]\n",
    "  for i in range(0,30):\n",
    "    a,b,c,d=generating_samples(x,y)\n",
    "    list_input_data.append(a)\n",
    "    list_output_data.append(b)\n",
    "    list_selected_row.append(c)\n",
    "    list_selected_columns.append(d)\n",
    "  from sklearn.tree import DecisionTreeRegressor\n",
    "  list_of_all_models=[]\n",
    "  for i in range(len(list_input_data)):\n",
    "    model=DecisionTreeRegressor()\n",
    "    model.fit(list_input_data[i],list_output_data[i])\n",
    "    list_of_all_models.append(model)\n",
    "  predicted_values=[]\n",
    "  for i in range(len(list_of_all_models)):\n",
    "    predicted_values.append(list_of_all_models[i].predict(x[:,list_selected_columns[i]]))\n",
    "  predict=np.median(np.array(predicted_values),axis=0)\n",
    "  #mean squared error\n",
    "  mse_full.append(mean_squared_error(y,predict))\n",
    "  obb_predict=[]\n",
    "  oob_score=[]\n",
    "  for i in range(len(x)):\n",
    "      predict_oob=[]\n",
    "      for j in range(len(list_of_all_models)):\n",
    "          if i not in list_selected_row[j]:\n",
    "              predict_oob.append(list_of_all_models[j].predict(x[i][list_selected_columns[j]].reshape(1,-1)))\n",
    "      oob_score.append(np.median(predict_oob))\n",
    "  #print(\"oob_score is : \",(mean_squared_error(y,oob_score)))\n",
    "  obb_score_full.append(mean_squared_error(y,oob_score))\n",
    "obb_score_full=np.array(obb_score_full)\n",
    "mse_full=np.array(mse_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1359,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kg1RR8toWwPv",
    "outputId": "5bafce90-9af2-4b95-e0a7-74e12542f0db"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "median OBB score which ran for 35 times : 14.934113615079267\n"
     ]
    }
   ],
   "source": [
    "print('median OBB score which ran for 35 times :',np.median(obb_score_full))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1360,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "YsN5WDozXOyI",
    "outputId": "d04f46a1-b5ec-4c5a-b388-62bce29273f7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "median mean squared error which ran for 35 times : 0.09909437284147297\n"
     ]
    }
   ],
   "source": [
    "print('median mean squared error which ran for 35 times :',np.median(mse_full))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1373,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 283
    },
    "id": "A9Vnfy9CZex-",
    "outputId": "4d33ae2e-5fe0-451d-e3ad-c2753931d7c4"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAPp0lEQVR4nO3db4xcV33G8e+DA2kLVE3kTWpsq2siIzWphEFbFykqAtKSNEg4qEplVCG/iGRUJS1I0MoBqaSqLKVVgVcNlRERVguklvgTt0EtwaJFSFXCJso/J7i4xMQbu/YCrYA3bu38+mJvYOrMemdndjxjzvcjjebec8+Z+9sj+9nre+9cp6qQJP3se9mkC5AkXRwGviQ1wsCXpEYY+JLUCANfkhpx2aQLAFi/fn3Nzs5OugxJuqQ88sgj36uqmUH7T0Xgz87OMj8/P+kyJOmSkuS7q+nvKR1JaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWrEVHzTVlrJ7J4HJrbvY3e/Y2L7ltaSR/iS1AgDX5IaYeBLUiMMfElqxIqBn+Tnkjyc5PEkh5P8Wdd+ZZIHk3y7e7+iZ8ydSY4mOZLkxnH+AJKkwQxyhH8GeFtVvR7YBtyU5E3AHuBQVW0FDnXrJLkW2AlcB9wE3JNk3RhqlyStwoqBX0t+3K2+vHsVsAPY37XvB27plncA91XVmap6FjgKbF/LoiVJqzfQOfwk65I8BpwGHqyqh4Crq+okQPd+Vdd9I3C8Z/hC13b+Z+5OMp9kfnFxcYQfQZI0iIECv6rOVdU2YBOwPcmvXaB7+n1En8/cV1VzVTU3MzPwf8koSRrSqu7Sqar/Bv6FpXPzp5JsAOjeT3fdFoDNPcM2ASdGLVSSNJpB7tKZSfJL3fLPA78FfAs4COzquu0C7u+WDwI7k1yeZAuwFXh4jeuWJK3SIM/S2QDs7+60eRlwoKr+Mcm/AQeS3AY8B9wKUFWHkxwAngbOArdX1bnxlC9JGtSKgV9VTwBv6NP+feCGZcbsBfaOXJ0kac34TVtJaoSPR5amlI+E1lrzCF+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEZdNugBp2s3ueWDSJUhrYsUj/CSbk3wtyTNJDid5X9d+V5LnkzzWvW7uGXNnkqNJjiS5cZw/gCRpMIMc4Z8FPlBVjyZ5NfBIkge7bR+vqr/q7ZzkWmAncB3wGuCrSV5XVefWsnBJ0uqseIRfVSer6tFu+UfAM8DGCwzZAdxXVWeq6lngKLB9LYqVJA1vVRdtk8wCbwAe6pruSPJEknuTXNG1bQSO9wxboM8viCS7k8wnmV9cXFx95ZKkVRk48JO8Cvg88P6q+iHwCeAaYBtwEvjoi137DK+XNFTtq6q5qpqbmZlZbd2SpFUaKPCTvJylsP9MVX0BoKpOVdW5qnoB+CQ/PW2zAGzuGb4JOLF2JUuShjHIXToBPgU8U1Uf62nf0NPtXcBT3fJBYGeSy5NsAbYCD69dyZKkYQxyl871wHuAJ5M81rV9CHh3km0sna45BrwXoKoOJzkAPM3SHT63e4eOJE3eioFfVd+g/3n5L19gzF5g7wh1SZLWmI9WkKRGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNWLFwE+yOcnXkjyT5HCS93XtVyZ5MMm3u/cresbcmeRokiNJbhznDyBJGswgR/hngQ9U1a8CbwJuT3ItsAc4VFVbgUPdOt22ncB1wE3APUnWjaN4SdLgVgz8qjpZVY92yz8CngE2AjuA/V23/cAt3fIO4L6qOlNVzwJHge1rXLckaZVWdQ4/ySzwBuAh4OqqOglLvxSAq7puG4HjPcMWurbzP2t3kvkk84uLi0OULklajYEDP8mrgM8D76+qH16oa5+2eklD1b6qmququZmZmUHLkCQNaaDAT/JylsL+M1X1ha75VJIN3fYNwOmufQHY3DN8E3BibcqVJA1rkLt0AnwKeKaqPtaz6SCwq1veBdzf074zyeVJtgBbgYfXrmRJ0jAuG6DP9cB7gCeTPNa1fQi4GziQ5DbgOeBWgKo6nOQA8DRLd/jcXlXn1rpwSdLqrBj4VfUN+p+XB7hhmTF7gb0j1CVJWmN+01aSGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktSIFQM/yb1JTid5qqftriTPJ3mse93cs+3OJEeTHEly47gKlyStziBH+J8GburT/vGq2ta9vgyQ5FpgJ3BdN+aeJOvWqlhJ0vBWDPyq+jrwgwE/bwdwX1WdqapngaPA9hHqkyStkVHO4d+R5InulM8VXdtG4HhPn4WuTZI0YcMG/ieAa4BtwEngo117+vStfh+QZHeS+STzi4uLQ5YhSRrUUIFfVaeq6lxVvQB8kp+etlkANvd03QScWOYz9lXVXFXNzczMDFOGJGkVhgr8JBt6Vt8FvHgHz0FgZ5LLk2wBtgIPj1aiJGktXLZShySfA94CrE+yAHwEeEuSbSydrjkGvBegqg4nOQA8DZwFbq+qc2OpXJK0KisGflW9u0/zpy7Qfy+wd5SiJElrz2/aSlIjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqxIr/xaEkXSyzex6YyH6P3f2Oiez3YvMIX5IaYeBLUiMMfElqhIEvSY0w8CWpESsGfpJ7k5xO8lRP25VJHkzy7e79ip5tdyY5muRIkhvHVbgkaXUGOcL/NHDTeW17gENVtRU41K2T5FpgJ3BdN+aeJOvWrFpJ0tBWDPyq+jrwg/OadwD7u+X9wC097fdV1ZmqehY4Cmxfm1IlSaMY9hz+1VV1EqB7v6pr3wgc7+m30LVJkiZsrS/apk9b9e2Y7E4yn2R+cXFxjcuQJJ1v2MA/lWQDQPd+umtfADb39NsEnOj3AVW1r6rmqmpuZmZmyDIkSYMaNvAPAru65V3A/T3tO5NcnmQLsBV4eLQSJUlrYcWHpyX5HPAWYH2SBeAjwN3AgSS3Ac8BtwJU1eEkB4CngbPA7VV1bky1S5JWYcXAr6p3L7PphmX67wX2jlKUJGnt+U1bSWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWrEZaMMTnIM+BFwDjhbVXNJrgT+HpgFjgG/V1X/NVqZkqRRrcUR/luraltVzXXre4BDVbUVONStS5ImbByndHYA+7vl/cAtY9iHJGmVRg38Ar6S5JEku7u2q6vqJED3flW/gUl2J5lPMr+4uDhiGZKklYx0Dh+4vqpOJLkKeDDJtwYdWFX7gH0Ac3NzNWIdkqQVjHSEX1UnuvfTwBeB7cCpJBsAuvfToxYpSRrd0IGf5JVJXv3iMvB24CngILCr67YLuH/UIiVJoxvllM7VwBeTvPg5n62qf0ryTeBAktuA54BbRy9TkjSqoQO/qr4DvL5P+/eBG0YpSpK09vymrSQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRozyf9qqQbN7Hph0CZKG5BG+JDXCwJekRhj4ktQIA1+SGuFF20uQF04lDWNsR/hJbkpyJMnRJHvGtR9J0mDGcoSfZB3w18BvAwvAN5McrKqnx7G/SfFIW/rZMMm/y8fufsdF29e4TulsB45W1XcAktwH7ADGEvgGryStbFyBvxE43rO+APxGb4cku4Hd3eqPkxwZUy3jth743qSLGJK1T8bU156/WHbT1Nd+AVNZ+wXmutdytf/KavY1rsBPn7b6fytV+4B9Y9r/RZNkvqrmJl3HMKx9Mqx9Mqx9fBdtF4DNPeubgBNj2pckaQDjCvxvAluTbEnyCmAncHBM+5IkDWAsp3Sq6mySO4B/BtYB91bV4XHsawpcyqelrH0yrH0ymq89VbVyL0nSJc9HK0hSIwx8SWqEgb+MJPcmOZ3kqZ62P0/yRJLHknwlyWuWGXssyZNdv/mLV/VP9v+S2nu2fTBJJVm/zNiJPhJjxNqnbt6T3JXk+a6mx5LcvMzYqZv3VdQ+dfPetf9hN6eHk/zlMmOnbt679kFqX/28V5WvPi/gzcAbgad62n6xZ/mPgL9ZZuwxYP001d61b2bpQvp3+9XH0gX2/wBeC7wCeBy49lKofVrnHbgL+OAK46Zy3gepfYrn/a3AV4HLu/WrLqF5X7H2YefdI/xlVNXXgR+c1/bDntVXct6XyaZFv9o7Hwf+hOXr/skjMarqf4AXH4lx0YxQ+8RdoPaVTPO8T71lav8D4O6qOtP1Od1n6LTO+yC1D8XAX6Uke5McB34f+NNluhXwlSSPdI+QmLgk7wSer6rHL9Ct3yMxNo61sAEMWDtM4bx37uhOBd6b5Io+26dy3jsr1Q7TOe+vA34zyUNJ/jXJr/fpM63zPkjtMMS8G/irVFUfrqrNwGeAO5bpdn1VvRH4HeD2JG++aAX2keQXgA+z/C+on3Tt0zbRI+pV1A5TNu+dTwDXANuAk8BH+/SZunnvDFI7TOe8XwZcAbwJ+GPgQJLz53la532Q2mGIeTfwh/dZ4Hf7baiqE937aeCLLP3TcZKuAbYAjyc5xtKjLh5N8svn9ZvGR2IMWvs0zjtVdaqqzlXVC8Anl6lpGud90Nqnct5ZmtMv1JKHgRdYegDZ+X2mbt4ZrPah5t3AX4UkW3tW3wl8q0+fVyZ59YvLwNuBl9xxcjFV1ZNVdVVVzVbVLEt/oN5YVf95XtepeyTGoLVP47x3tWzoWX0X/WuaunmHwWqf1nkHvgS8DSDJ61i6KHv+0yanct4ZoPah5/1iXpG+lF7A51j6Z+z/shQytwGf7yb1CeAfgI1d39cAX+6WX8vS1f7HgcPAh6eh9vO2H6O7ut9be7d+M/DvLN29cMnUPq3zDvwt8GT3Z+YgsOFSmfdBap/ieX8F8Hfd39dHgbddQvO+Yu3DzruPVpCkRnhKR5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRvwfHB2cG2bxj4oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95.0 confidence interval 14.6 and 15.9 of OBB score\n"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "from pandas import read_csv\n",
    "from sklearn.utils import resample\n",
    "from sklearn.metrics import accuracy_score\n",
    "from matplotlib import pyplot\n",
    "t = np.array(obb_score_full)\n",
    "\n",
    "# configure bootstrap\n",
    "n_iterations = 1000\n",
    "n_size = int(len(t))\n",
    "\n",
    "# run bootstrap\n",
    "medians = list()\n",
    "for i in range(n_iterations):\n",
    "    # prepare train and test sets\n",
    "    s = resample(t, n_samples=n_size);\n",
    "    m = numpy.median(s);\n",
    "    #print(m)\n",
    "    medians.append(m)\n",
    "\n",
    "# plot scores\n",
    "pyplot.hist(medians)\n",
    "pyplot.show()\n",
    "\n",
    "# confidence intervals\n",
    "alpha = 0.95\n",
    "p = ((1.0-alpha)/2.0) * 100\n",
    "lower =  numpy.percentile(medians, p)\n",
    "\n",
    "p = (alpha+((1.0-alpha)/2.0)) * 100\n",
    "upper =  numpy.percentile(medians, p)\n",
    "print('%.1f confidence interval %.1f and %.1f of OBB score' % (alpha*100, lower, upper))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1374,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 283
    },
    "id": "0kHla_fibdO1",
    "outputId": "ab037790-0462-4a05-9d6c-5912982e1856"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD4CAYAAAAJmJb0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAARI0lEQVR4nO3df6yeZX3H8ffHgoi/AoQDq22zoqnbwMRizjo2lsWJGwSWFf9gqclcl7BUE0h00yzF/aH+0YQl/liWDJIqzM6prFEcDegmMo0xccCBIVJKZ5UODu3oUceA/VGlfPfHuZnP8Jw+z3l+9PRcvF/Jk+d+rvu67ud79Zx8zt3r3M99UlVIktr0suUuQJI0OYa8JDXMkJekhhnyktQwQ16SGnbKchcAcPbZZ9f69euXuwxJWlHuu+++H1bV1PH6nBQhv379emZmZpa7DElaUZL8R78+LtdIUsMMeUlqmCEvSQ0z5CWpYYa8JDXMkJekhhnyktQwQ16SGmbIS1LDTopPvEr9rN9+x7K998Hrr1i295ZG5Zm8JDXMkJekhhnyktQwQ16SGmbIS1LDDHlJapghL0kNM+QlqWF9Qz7JK5Lck+Q7SfYm+UjX/uEkTyR5oHtc3jPmuiQHkuxPcukkJyBJWtwgn3g9Crytqp5NcirwrSRf6fZ9oqo+2ts5yfnAFuAC4HXA15K8saqOjbNwSVJ/fc/ka96z3ctTu0cdZ8hm4JaqOlpVjwIHgE0jVypJWrKB1uSTrEryAHAEuLOq7u52XZvkwSQ3Jzmza1sDPN4zfLZre/ExtyWZSTIzNzc3/AwkSYsaKOSr6lhVbQTWApuSvAm4EXgDsBE4DHys656FDrHAMXdW1XRVTU9NTQ1RuiSpnyVdXVNVTwHfAC6rqie78H8e+CQ/W5KZBdb1DFsLHBq9VEnSUg1ydc1UkjO67dOBtwOPJFnd0+0dwEPd9h5gS5LTkpwHbADuGWvVkqSBDHJ1zWpgV5JVzP9Q2F1Vtyf5TJKNzC/FHATeDVBVe5PsBh4GngOu8coaSVoefUO+qh4ELlyg/V3HGbMD2DFaaZKkUfmJV0lqmCEvSQ0z5CWpYYa8JDXMkJekhhnyktQwQ16SGmbIS1LDDHlJapghL0kNM+QlqWGGvCQ1zJCXpIYZ8pLUMENekhpmyEtSwwx5SWqYIS9JDTPkJalhfUM+ySuS3JPkO0n2JvlI135WkjuTfK97PrNnzHVJDiTZn+TSSU5AkrS4Qc7kjwJvq6o3AxuBy5JcBGwH7qqqDcBd3WuSnA9sAS4ALgNuSLJqArVLkvroG/I179nu5ando4DNwK6ufRdwZbe9Gbilqo5W1aPAAWDTOIuWJA1moDX5JKuSPAAcAe6sqruBc6vqMED3fE7XfQ3weM/w2a7txcfclmQmyczc3NwIU5AkLWagkK+qY1W1EVgLbErypuN0z0KHWOCYO6tquqqmp6amBipWkrQ0S7q6pqqeAr7B/Fr7k0lWA3TPR7pus8C6nmFrgUOjFipJWrpBrq6ZSnJGt3068HbgEWAPsLXrthW4rdveA2xJclqS84ANwD1jrluSNIBTBuizGtjVXSHzMmB3Vd2e5NvA7iRXA48BVwFU1d4ku4GHgeeAa6rq2GTKlyQdT9+Qr6oHgQsXaP8RcMkiY3YAO0auTpI0Ej/xKkkNG2S5RieZ9dvvWLb3Pnj9Fcv23pKWzjN5SWqYIS9JDTPkJalhhrwkNcyQl6SGGfKS1DBDXpIaZshLUsMMeUlqmCEvSQ0z5CWpYYa8JDXMkJekhhnyktQwQ16SGmbIS1LDDHlJaljfkE+yLsnXk+xLsjfJe7v2Dyd5IskD3ePynjHXJTmQZH+SSyc5AUnS4gb583/PAe+vqvuTvAa4L8md3b5PVNVHezsnOR/YAlwAvA74WpI3VtWxcRYuSeqv75l8VR2uqvu77WeAfcCa4wzZDNxSVUer6lHgALBpHMVKkpZmSWvySdYDFwJ3d03XJnkwyc1Jzuza1gCP9wybZYEfCkm2JZlJMjM3N7f0yiVJfQ0c8kleDXwReF9VPQ3cCLwB2AgcBj72QtcFhtfPNVTtrKrpqpqemppaat2SpAEMFPJJTmU+4D9bVbcCVNWTVXWsqp4HPsnPlmRmgXU9w9cCh8ZXsiRpUINcXRPgJmBfVX28p311T7d3AA9123uALUlOS3IesAG4Z3wlS5IGNcjVNRcD7wK+m+SBru2DwDuTbGR+KeYg8G6AqtqbZDfwMPNX5lzjlTWStDz6hnxVfYuF19m/fJwxO4AdI9QlSRoDP/EqSQ0z5CWpYYa8JDXMkJekhhnyktQwQ16SGmbIS1LDDHlJapghL0kNM+QlqWGGvCQ1zJCXpIYZ8pLUMENekhpmyEtSwwx5SWqYIS9JDTPkJalhhrwkNaxvyCdZl+TrSfYl2ZvkvV37WUnuTPK97vnMnjHXJTmQZH+SSyc5AUnS4gY5k38OeH9V/QpwEXBNkvOB7cBdVbUBuKt7TbdvC3ABcBlwQ5JVkyheknR8fUO+qg5X1f3d9jPAPmANsBnY1XXbBVzZbW8Gbqmqo1X1KHAA2DTmuiVJA1jSmnyS9cCFwN3AuVV1GOZ/EADndN3WAI/3DJvt2l58rG1JZpLMzM3NDVG6JKmfgUM+yauBLwLvq6qnj9d1gbb6uYaqnVU1XVXTU1NTg5YhSVqCgUI+yanMB/xnq+rWrvnJJKu7/auBI137LLCuZ/ha4NB4ypUkLcUgV9cEuAnYV1Uf79m1B9jabW8Fbutp35LktCTnARuAe8ZXsiRpUKcM0Odi4F3Ad5M80LV9ELge2J3kauAx4CqAqtqbZDfwMPNX5lxTVcfGXbgkqb++IV9V32LhdXaASxYZswPYMUJdkqQx8BOvktQwQ16SGmbIS1LDDHlJapghL0kNM+QlqWGGvCQ1zJCXpIYZ8pLUMENekhpmyEtSwwx5SWqYIS9JDTPkJalhhrwkNcyQl6SGGfKS1DBDXpIaNsgf8r45yZEkD/W0fTjJE0ke6B6X9+y7LsmBJPuTXDqpwiVJ/Q1yJv9p4LIF2j9RVRu7x5cBkpwPbAEu6MbckGTVuIqVJC1N35Cvqm8CPx7weJuBW6rqaFU9ChwANo1QnyRpBKOsyV+b5MFuOefMrm0N8HhPn9muTZK0DIYN+RuBNwAbgcPAx7r2LNC3FjpAkm1JZpLMzM3NDVmGJOl4hgr5qnqyqo5V1fPAJ/nZkswssK6n61rg0CLH2FlV01U1PTU1NUwZkqQ+hgr5JKt7Xr4DeOHKmz3AliSnJTkP2ADcM1qJkqRhndKvQ5LPA28Fzk4yC3wIeGuSjcwvxRwE3g1QVXuT7AYeBp4DrqmqYxOpXJLUV9+Qr6p3LtB803H67wB2jFKUJGk8/MSrJDXMkJekhhnyktQwQ16SGmbIS1LD+l5dI/Vav/2O5S5B0hJ4Ji9JDTPkJalhhrwkNcyQl6SGGfKS1DBDXpIaZshLUsMMeUlqmCEvSQ0z5CWpYYa8JDXMkJekhhnyktSwviGf5OYkR5I81NN2VpI7k3yvez6zZ991SQ4k2Z/k0kkVLknqb5Az+U8Dl72obTtwV1VtAO7qXpPkfGALcEE35oYkq8ZWrSRpSfqGfFV9E/jxi5o3A7u67V3AlT3tt1TV0ap6FDgAbBpPqZKkpRp2Tf7cqjoM0D2f07WvAR7v6Tfbtf2cJNuSzCSZmZubG7IMSdLxjPsXr1mgrRbqWFU7q2q6qqanpqbGXIYkCYYP+SeTrAbono907bPAup5+a4FDw5cnSRrFsCG/B9jabW8Fbutp35LktCTnARuAe0YrUZI0rL5/yDvJ54G3AmcnmQU+BFwP7E5yNfAYcBVAVe1Nsht4GHgOuKaqjk2odklSH31DvqreuciuSxbpvwPYMUpRkqTx8BOvktQwQ16SGmbIS1LDDHlJapghL0kNM+QlqWGGvCQ1zJCXpIYZ8pLUMENekhpmyEtSwwx5SWpY3xuUSVoe67ffsWzvffD6K5btvTVenslLUsMMeUlqmMs1Uh/LuWwijcozeUlqmCEvSQ0z5CWpYSOtySc5CDwDHAOeq6rpJGcB/wCsBw4Cf1BV/zVamZKkYYzjTP63q2pjVU13r7cDd1XVBuCu7rUkaRlMYrlmM7Cr294FXDmB95AkDWDUkC/gq0nuS7Ktazu3qg4DdM/nLDQwybYkM0lm5ubmRixDkrSQUa+Tv7iqDiU5B7gzySODDqyqncBOgOnp6RqxDknSAkY6k6+qQ93zEeBLwCbgySSrAbrnI6MWKUkaztAhn+RVSV7zwjbwu8BDwB5ga9dtK3DbqEVKkoYzynLNucCXkrxwnM9V1T8luRfYneRq4DHgqtHLlCQNY+iQr6ofAG9eoP1HwCWjFCVpeS3X/Xq8xfH4+YlXSWqYIS9JDTPkJalhhrwkNcyQl6SGGfKS1DBDXpIaZshLUsP8Q94j8A88SzrZeSYvSQ0z5CWpYYa8JDXMkJekhhnyktQwQ16SGmbIS1LDDHlJapghL0kN8xOvkk4a/tnB8ZvYmXySy5LsT3IgyfZJvY8kaXETOZNPsgr4G+B3gFng3iR7qurhSbyf95CRpIVNarlmE3Cgqn4AkOQWYDMwkZCXpFEs54nipJeKJhXya4DHe17PAr/W2yHJNmBb9/LZJPsnUMfZwA8ncNyTmXN+aXDOjchfHnd3vzn/Yr/jTyrks0Bb/b8XVTuBnRN6//kikpmqmp7ke5xsnPNLg3N+aRjHnCf1i9dZYF3P67XAoQm9lyRpEZMK+XuBDUnOS/JyYAuwZ0LvJUlaxESWa6rquSTXAv8MrAJurqq9k3ivPia6HHSScs4vDc75pWHkOaeq+veSJK1I3tZAkhpmyEtSw1ZsyPe7bULm/XW3/8Ekb+nZd0aSLyR5JMm+JL9+Yqsfzohz/tMke5M8lOTzSV5xYqsfzgBz/uUk305yNMkHljL2ZDTsfJOsS/L17vt5b5L3ntjKhzfK17jbvyrJvyW5/cRUPLoRv6+Xll9VteIezP8y9/vA64GXA98Bzn9Rn8uBrzB/zf5FwN09+3YBf9Jtvxw4Y7nnNMk5M//htEeB07vXu4E/Xu45jWnO5wC/CuwAPrCUsSfbY8T5rgbe0m2/Bvj3k32+o865Z/+fAZ8Dbl/u+ZyIOS81v1bqmfz/3Tahqn4CvHDbhF6bgb+ref8KnJFkdZLXAr8F3ARQVT+pqqdOYO3DGnrO3b5TgNOTnAK8kpXxuYW+c66qI1V1L/DTpY49CQ0936o6XFX3d9vPAPuY/+F+shvla0yStcAVwKdORLFjMvSch8mvlRryC9024cXf0Iv1eT0wB/xt91+8TyV51SSLHZOh51xVTwAfBR4DDgP/XVVfnWCt4zLInCcxdrmMpeYk64ELgbvHU9ZEjTrnvwL+HHh+jDVN2ihzXnJ+rdSQ73vbhOP0OQV4C3BjVV0I/A+wEtZrh55zkjOZP1M4D3gd8Kokfzjm+iZhkDlPYuxyGbnmJK8Gvgi8r6qeHktVkzX0nJP8HnCkqu4bb0kTN8rXecn5tVJDfpDbJizWZxaYraoXznK+wPw/2slulDm/HXi0quaq6qfArcBvTLDWcRnl9hgr8dYaI9Wc5FTmA/6zVXXrmGublFHmfDHw+0kOMr/k8bYkfz/e8iZi1O/rJeXXSg35QW6bsAf4o+6Kk4uYX6I4XFX/CTye5Je6fpewMm6BPPScmV+muSjJK5OE+TnvO5HFD2mU22OsxFtrDF1z93W9CdhXVR+fYI3jNvScq+q6qlpbVeu7cf9SVSvhf6ijzHnp+bXcv2ke4TfUlzN/BcH3gb/o2t4DvKfbDvN/uOT7wHeB6Z6xG4EZ4EHgH4Ezl3s+J2DOHwEeAR4CPgOcttzzGdOcf4H5s5ungae67dcuNvZkfww7X+A3mf8v/4PAA93j8uWez6S/xj3HeCsr5OqaUee81PzytgaS1LCVulwjSRqAIS9JDTPkJalhhrwkNcyQl6SGGfKS1DBDXpIa9r/Fbfr00M6fhwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95.0 confidence interval 0.1 and 0.1 for mean squared error\n"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "from pandas import read_csv\n",
    "from sklearn.utils import resample\n",
    "from sklearn.metrics import accuracy_score\n",
    "from matplotlib import pyplot\n",
    "t = np.array(mse_full)\n",
    "\n",
    "# configure bootstrap\n",
    "n_iterations = 1000\n",
    "n_size = int(len(t))\n",
    "\n",
    "# run bootstrap\n",
    "medians = list()\n",
    "for i in range(n_iterations):\n",
    "    # prepare train and test sets\n",
    "    s = resample(t, n_samples=n_size);\n",
    "    m = numpy.median(s);\n",
    "    #print(m)\n",
    "    medians.append(m)\n",
    "\n",
    "# plot scores\n",
    "pyplot.hist(medians)\n",
    "pyplot.show()\n",
    "\n",
    "# confidence intervals\n",
    "alpha = 0.95\n",
    "p = ((1.0-alpha)/2.0) * 100\n",
    "lower =  numpy.percentile(medians, p)\n",
    "\n",
    "p = (alpha+((1.0-alpha)/2.0)) * 100\n",
    "upper =  numpy.percentile(medians, p)\n",
    "print('%.1f confidence interval %.1f and %.1f for mean squared error' % (alpha*100, lower, upper))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1362,
   "metadata": {
    "id": "iOh5dr-ibh7l"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jKTnJdiBVS_e"
   },
   "source": [
    "# <font color='blue'><b>Task 3</b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eXxrvZqHV1Fr"
   },
   "source": [
    "<font color='orange'><b>Flowchart for Task 3</b></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "NyjwEJ62V6a6"
   },
   "source": [
    "<b>Hint: </b> We created 30 models by using 30 samples in TASK-1. Here, we need send query point \"xq\"  to 30 models and perform the regression on the output generated by 30 models."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0emSwLL7VurD"
   },
   "source": [
    "![alt text](https://i.imgur.com/Y5cNhQk.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "29hjwKlWWDfo"
   },
   "source": [
    "*  <font color='blue'><b> code for TASK 3 </b></font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1363,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "i_pUlSD-VYD1",
    "outputId": "0debcbd1-68f7-4d87-fd52-02eed6a88ae1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "predicted value for xq is : 19.049999999999997\n"
     ]
    }
   ],
   "source": [
    "pred_y=[]\n",
    "xq= [0.18,20.0,5.00,0.0,0.421,5.60,72.2,7.median95,7.0,30.0,19.1,372.13,18.60]\n",
    "for i in range(len(list_of_all_models)):\n",
    "  pred_y.append(list_of_all_models[i].predict(np.array(xq)[list_selected_columns[i]].reshape(1,-1))[0])\n",
    "print('predicted value for xq is : {}'.format(np.median(pred_y)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DJHTGEZgWJjR"
   },
   "source": [
    "<br><br><br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IOdUi-0xWOJ9"
   },
   "source": [
    "<font color='red'><b>observations for task 1, task 2, task 3</b></font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1364,
   "metadata": {
    "id": "AIcax45hWKT-"
   },
   "outputs": [],
   "source": [
    "from tabulate import tabulate\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1375,
   "metadata": {
    "id": "wmA7TLMzt4SL"
   },
   "outputs": [],
   "source": [
    "table = [[\"Task 1 : mean squared error \",0.19131052371541513],[\"Task 1: OBB score\",12.826694419596326],[\"Task 2 : median OBB score which ran for 35 times\",14.95],[\"Task 2 : median mean squared error which ran for 35 times\",0.09909437284147297],[\"Task 2 :95.0 confidence interval of OBB score\",\"14.4 and 15.9\"],[\"Task 2: 95.0 confidence interval for mean squared error\",'0.1 and 0.1'],[\"Task 3 : Predicted value for given data is\",19.049999999999997]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1376,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kqGW1--avDxp",
    "outputId": "76baf2a9-c22c-47f0-82fd-8f928d6f5509"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OBSERVATION\n",
      "---------------------------------------------------------  -------------------\n",
      "Task 1 : mean squared error                                0.19131052371541513\n",
      "Task 1: OBB score                                          12.826694419596326\n",
      "Task 2 : median OBB score which ran for 35 times           14.95\n",
      "Task 2 : median mean squared error which ran for 35 times  0.09909437284147297\n",
      "Task 2 :95.0 confidence interval of OBB score              14.4 and 15.9\n",
      "Task 2: 95.0 confidence interval for mean squared error    0.1 and 0.1\n",
      "Task 3 : Predicted value for given data is                 19.049999999999997\n",
      "---------------------------------------------------------  -------------------\n"
     ]
    }
   ],
   "source": [
    "print('OBSERVATION')\n",
    "print(tabulate(table))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1366,
   "metadata": {
    "id": "s_WNVLWQvmjL"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Copy of Bootstrap_assignment_modified.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0249e86337c340afb12485dc1471647c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "0c21d0cbb6f044afa756e9c7f08f78e4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "165ea5d2cd6443739e8d004bb13c9bea": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_72396c4af831487fb3cbc01be369a06c",
      "max": 35,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_0249e86337c340afb12485dc1471647c",
      "value": 35
     }
    },
    "32d833a3473a4b66914702bd214dc121": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4b5257a8e2594d8b8fe977d726c06bf5": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "72396c4af831487fb3cbc01be369a06c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8bc496db1ff24bb2a47df4c988de1268": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9284e14a76954cb48d3ef795f44f0e3f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4b5257a8e2594d8b8fe977d726c06bf5",
      "placeholder": "​",
      "style": "IPY_MODEL_0c21d0cbb6f044afa756e9c7f08f78e4",
      "value": " 35/35 [00:27&lt;00:00,  1.27it/s]"
     }
    },
    "bff3c595a11044deb8f21bc6e8e6086d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8bc496db1ff24bb2a47df4c988de1268",
      "placeholder": "​",
      "style": "IPY_MODEL_f12ec296ae444149a1b5bf88e271c20e",
      "value": "100%"
     }
    },
    "f12ec296ae444149a1b5bf88e271c20e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "fef1d524c5884f69b953d1fc288b46b2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_bff3c595a11044deb8f21bc6e8e6086d",
       "IPY_MODEL_165ea5d2cd6443739e8d004bb13c9bea",
       "IPY_MODEL_9284e14a76954cb48d3ef795f44f0e3f"
      ],
      "layout": "IPY_MODEL_32d833a3473a4b66914702bd214dc121"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}