03102022_BayesianOpt(DDN+OneHot)_AlphaD3M.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "17534b9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import math\n",
    "import numpy as np\n",
    "import time\n",
    "from collections import defaultdict\n",
    "\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
    "from sklearn.neural_network import MLPRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a2c8c0df",
   "metadata": {},
   "outputs": [],
   "source": [
    "AutoML_path = \"\"\n",
    "log_folder = \"alphad3m_log\"\n",
    "\n",
    "log_DDN = AutoML_path+log_folder+'/DDN_single_time'\n",
    "log_pipeline_steps = AutoML_path+log_folder+'/pipeline_steps_class_02272022'\n",
    "log_uniq_prim = AutoML_path+log_folder+'/uniq_prim_02272022'\n",
    "\n",
    "combination_name = 'DDN+BO'\n",
    "today = '03102022'\n",
    "log_target_test_pred_time = AutoML_path+log_folder+'/'+combination_name+'_target_test_pred_time_'+today\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "62870f07",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39\n",
      "7811\n"
     ]
    }
   ],
   "source": [
    "DDN = pickle.load(open(log_DDN, \"rb\"))\n",
    "print(len(DDN))\n",
    "\n",
    "pipeline_steps_class = pickle.load(open(log_pipeline_steps, 'rb'))\n",
    "print(len(pipeline_steps_class))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cc087516",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.gaussian_process.kernels import Matern\n",
    "from sklearn.gaussian_process import GaussianProcessRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "65dba102",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========================================\n",
      "0 adult\n",
      "X_train shape: (1526, 37) y_train shape: (1526,)\n",
      "X_test shape: (452, 37) y_test shape: (452,)\n",
      "Threshold 0.05 MAE: 0.06972411392660854 MSE: 0.02031647724703847\n",
      "Time: 0.5723600387573242\n",
      "X_train shape: (3405, 37) y_train shape: (3405,)\n",
      "X_test shape: (452, 37) y_test shape: (452,)\n",
      "Threshold 0.1 MAE: 0.06889635764205287 MSE: 0.02015159877372517\n",
      "Time: 3.8732473850250244\n",
      "X_train shape: (5415, 37) y_train shape: (5415,)\n",
      "X_test shape: (452, 37) y_test shape: (452,)\n",
      "Threshold 0.15000000000000002 MAE: 0.07354906064871104 MSE: 0.018370969061430337\n",
      "Time: 14.89076280593872\n",
      "X_train shape: (5892, 37) y_train shape: (5892,)\n",
      "X_test shape: (452, 37) y_test shape: (452,)\n",
      "Threshold 0.2 MAE: 0.06742265699158145 MSE: 0.018840165979025002\n",
      "Time: 20.001418113708496\n",
      "X_train shape: (6745, 37) y_train shape: (6745,)\n",
      "X_test shape: (452, 37) y_test shape: (452,)\n",
      "Threshold 0.25 MAE: 0.08663327498346372 MSE: 0.018117160395707873\n",
      "Time: 26.533244371414185\n",
      "X_train shape: (6931, 37) y_train shape: (6931,)\n",
      "X_test shape: (452, 37) y_test shape: (452,)\n",
      "Threshold 0.3 MAE: 0.08957468543172284 MSE: 0.018170390138371365\n",
      "Time: 27.895576000213623\n",
      "X_train shape: (7147, 37) y_train shape: (7147,)\n",
      "X_test shape: (452, 37) y_test shape: (452,)\n",
      "Threshold 0.35000000000000003 MAE: 0.09370789218459374 MSE: 0.018307540293091163\n",
      "Time: 32.191689014434814\n",
      "adult_DDN+BO_0.35 adult_DDN+BO_0.4\n",
      "36 36\n",
      "adult_DDN+BO_0.4 adult_DDN+BO_0.45\n",
      "36 36\n",
      "========================================\n",
      "1 airlines\n",
      "X_train shape: (5, 37) y_train shape: (5,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.05 MAE: 0.023471008883863397 MSE: 0.0012414114518542554\n",
      "Time: 0.024281978607177734\n",
      "X_train shape: (1074, 37) y_train shape: (1074,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.1 MAE: 0.052748281839974535 MSE: 0.003079260711372984\n",
      "Time: 0.4872281551361084\n",
      "X_train shape: (1494, 37) y_train shape: (1494,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.15000000000000002 MAE: 0.0732076026161595 MSE: 0.005979212975547229\n",
      "Time: 0.7343590259552002\n",
      "X_train shape: (2992, 37) y_train shape: (2992,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.2 MAE: 0.018145336505677664 MSE: 0.0009433206063499964\n",
      "Time: 3.4115400314331055\n",
      "airlines_DDN+BO_0.2 airlines_DDN+BO_0.25\n",
      "16 16\n",
      "X_train shape: (5174, 37) y_train shape: (5174,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.3 MAE: 0.07005883720563427 MSE: 0.005688570788733828\n",
      "Time: 13.96946406364441\n",
      "X_train shape: (5583, 37) y_train shape: (5583,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.35000000000000003 MAE: 0.08734568455151753 MSE: 0.008409598727901469\n",
      "Time: 16.498345136642456\n",
      "X_train shape: (6859, 37) y_train shape: (6859,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.4 MAE: 0.12387364576866661 MSE: 0.01612501023414936\n",
      "Time: 28.522095918655396\n",
      "X_train shape: (7630, 37) y_train shape: (7630,)\n",
      "X_test shape: (181, 37) y_test shape: (181,)\n",
      "Threshold 0.45 MAE: 0.14317861985274644 MSE: 0.02128044730106554\n",
      "Time: 33.031338930130005\n",
      "========================================\n",
      "2 albert\n",
      "X_train shape: (397, 37) y_train shape: (397,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.05 MAE: 0.03828555551677304 MSE: 0.0016933317800992792\n",
      "Time: 0.2158663272857666\n",
      "X_train shape: (1250, 37) y_train shape: (1250,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.1 MAE: 0.06975750769230774 MSE: 0.005884109879442373\n",
      "Time: 0.43758511543273926\n",
      "X_train shape: (2102, 37) y_train shape: (2102,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.15000000000000002 MAE: 0.06923895197448751 MSE: 0.005812032470525389\n",
      "Time: 1.4433317184448242\n",
      "X_train shape: (3168, 37) y_train shape: (3168,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.2 MAE: 0.034086980874392214 MSE: 0.0012881060817155055\n",
      "Time: 3.519240140914917\n",
      "albert_DDN+BO_0.2 albert_DDN+BO_0.25\n",
      "16 16\n",
      "X_train shape: (5350, 37) y_train shape: (5350,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.3 MAE: 0.0392799976145539 MSE: 0.00256091821259936\n",
      "Time: 15.02866792678833\n",
      "X_train shape: (6171, 37) y_train shape: (6171,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.35000000000000003 MAE: 0.0691049812169365 MSE: 0.0057934984289931485\n",
      "Time: 22.228327751159668\n",
      "X_train shape: (7095, 37) y_train shape: (7095,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.4 MAE: 0.0933485326412565 MSE: 0.009731948546275734\n",
      "Time: 30.10265016555786\n",
      "X_train shape: (7806, 37) y_train shape: (7806,)\n",
      "X_test shape: (5, 37) y_test shape: (5,)\n",
      "Threshold 0.45 MAE: 0.11148467064946385 MSE: 0.01344683178981943\n",
      "Time: 37.39940810203552\n",
      "========================================\n",
      "3 Amazon_employee_access\n",
      "X_train shape: (1763, 37) y_train shape: (1763,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.10708386475696507 MSE: 0.013818947463047439\n",
      "Time: 1.0372989177703857\n",
      "X_train shape: (3370, 37) y_train shape: (3370,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.12092858070085742 MSE: 0.016273105803188723\n",
      "Time: 4.568525075912476\n",
      "X_train shape: (4218, 37) y_train shape: (4218,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.13555714606578523 MSE: 0.019460272004862313\n",
      "Time: 8.53882098197937\n",
      "X_train shape: (5060, 37) y_train shape: (5060,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.1505063834563972 MSE: 0.023346500475191025\n",
      "Time: 13.000951766967773\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.18863765388205317 MSE: 0.03602613122593181\n",
      "Time: 24.30071187019348\n",
      "Amazon_employee_access_DDN+BO_0.25 Amazon_employee_access_DDN+BO_0.3\n",
      "28 28\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.21340295989079644 MSE: 0.04633409854982407\n",
      "Time: 30.63935089111328\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.21689529531410182 MSE: 0.047923513400259084\n",
      "Time: 29.255952835083008\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.22177133505368468 MSE: 0.050198904657244206\n",
      "Time: 32.99967384338379\n",
      "========================================\n",
      "4 APSFailure\n",
      "X_train shape: (1617, 37) y_train shape: (1617,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.05 MAE: 0.12723974438260136 MSE: 0.016230208723389247\n",
      "Time: 0.8022429943084717\n",
      "X_train shape: (2438, 37) y_train shape: (2438,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.1 MAE: 0.12945564822410596 MSE: 0.016799021029962984\n",
      "Time: 2.1815907955169678\n",
      "X_train shape: (4416, 37) y_train shape: (4416,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.15000000000000002 MAE: 0.17059455401417095 MSE: 0.029142758032133398\n",
      "Time: 9.747895002365112\n",
      "APSFailure_DDN+BO_0.15 APSFailure_DDN+BO_0.2\n",
      "20 20\n",
      "X_train shape: (5258, 37) y_train shape: (5258,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.25 MAE: 0.18849304667467998 MSE: 0.0355698848175426\n",
      "Time: 14.236863136291504\n",
      "X_train shape: (6326, 37) y_train shape: (6326,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.3 MAE: 0.23303461879815382 MSE: 0.054345389731240396\n",
      "Time: 25.00802183151245\n",
      "X_train shape: (6343, 37) y_train shape: (6343,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.35000000000000003 MAE: 0.23339566778170331 MSE: 0.05451379391210674\n",
      "Time: 24.291059970855713\n",
      "X_train shape: (7179, 37) y_train shape: (7179,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.4 MAE: 0.26188867914430297 MSE: 0.06862593643678717\n",
      "Time: 30.510320901870728\n",
      "X_train shape: (7365, 37) y_train shape: (7365,)\n",
      "X_test shape: (18, 37) y_test shape: (18,)\n",
      "Threshold 0.45 MAE: 0.26600699175066095 MSE: 0.07079997583307569\n",
      "Time: 31.281415939331055\n",
      "========================================\n",
      "5 Australian\n",
      "X_train shape: (2171, 37) y_train shape: (2171,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.08750518536185875 MSE: 0.009327310118355569\n",
      "Time: 1.5034599304199219\n",
      "X_train shape: (3447, 37) y_train shape: (3447,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.06647355021684818 MSE: 0.007042702371163441\n",
      "Time: 4.367039203643799\n",
      "X_train shape: (5708, 37) y_train shape: (5708,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.0893868272928998 MSE: 0.009590119070502364\n",
      "Time: 17.219241857528687\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.10971711153120675 MSE: 0.01305195260024042\n",
      "Time: 22.694273948669434\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.13498690154282178 MSE: 0.018882954120427738\n",
      "Time: 30.04792618751526\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.13860799083276923 MSE: 0.01984768078312371\n",
      "Time: 32.16665482521057\n",
      "Australian_DDN+BO_0.3 Australian_DDN+BO_0.35\n",
      "35 35\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.1436480457575848 MSE: 0.02124807058352602\n",
      "Time: 31.363654851913452\n",
      "Australian_DDN+BO_0.4 Australian_DDN+BO_0.45\n",
      "36 36\n",
      "========================================\n",
      "6 bank-marketing\n",
      "X_train shape: (1758, 37) y_train shape: (1758,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.05 MAE: 0.06667626793750361 MSE: 0.008441991697636467\n",
      "Time: 0.9400160312652588\n",
      "X_train shape: (3759, 37) y_train shape: (3759,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.1 MAE: 0.07986375558244097 MSE: 0.009882430993200578\n",
      "Time: 5.448132038116455\n",
      "X_train shape: (4241, 37) y_train shape: (4241,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.15000000000000002 MAE: 0.08283393779336977 MSE: 0.010262749053504406\n",
      "Time: 8.374558925628662\n",
      "X_train shape: (5719, 37) y_train shape: (5719,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.2 MAE: 0.1366683173654656 MSE: 0.020610230844739574\n",
      "Time: 16.10707712173462\n",
      "X_train shape: (5935, 37) y_train shape: (5935,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.25 MAE: 0.13918197579250238 MSE: 0.02125295822527947\n",
      "Time: 18.947728872299194\n",
      "X_train shape: (6788, 37) y_train shape: (6788,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.3 MAE: 0.1669092508822774 MSE: 0.029266418635120774\n",
      "Time: 25.888073921203613\n",
      "X_train shape: (6974, 37) y_train shape: (6974,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.35000000000000003 MAE: 0.17080558734414156 MSE: 0.03052587597567888\n",
      "Time: 27.783759117126465\n",
      "X_train shape: (7190, 37) y_train shape: (7190,)\n",
      "X_test shape: (409, 37) y_test shape: (409,)\n",
      "Threshold 0.4 MAE: 0.17625887692103598 MSE: 0.032346146129111596\n",
      "Time: 29.41172218322754\n",
      "bank-marketing_DDN+BO_0.4 bank-marketing_DDN+BO_0.45\n",
      "36 36\n",
      "========================================\n",
      "7 blood-transfusion-service-center\n",
      "X_train shape: (1798, 37) y_train shape: (1798,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.05 MAE: 0.027841063327989888 MSE: 0.005783027662550204\n",
      "Time: 0.9786148071289062\n",
      "X_train shape: (3677, 37) y_train shape: (3677,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.1 MAE: 0.029165561371882198 MSE: 0.005920890623974192\n",
      "Time: 5.149893283843994\n",
      "X_train shape: (5471, 37) y_train shape: (5471,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.15000000000000002 MAE: 0.03821659475070257 MSE: 0.005236394992134578\n",
      "Time: 14.770866870880127\n",
      "X_train shape: (6181, 37) y_train shape: (6181,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.2 MAE: 0.028106771588762937 MSE: 0.005324679141067374\n",
      "Time: 20.20140790939331\n",
      "X_train shape: (7017, 37) y_train shape: (7017,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.25 MAE: 0.04865782130407433 MSE: 0.005562669420584263\n",
      "Time: 27.967429161071777\n",
      "X_train shape: (7203, 37) y_train shape: (7203,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.3 MAE: 0.05212666488377869 MSE: 0.0057381009004294735\n",
      "Time: 29.312274932861328\n",
      "X_train shape: (7419, 37) y_train shape: (7419,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.35000000000000003 MAE: 0.05699905358755324 MSE: 0.006040242156954701\n",
      "Time: 32.530975103378296\n",
      "blood-transfusion-service-center_DDN+BO_0.35 blood-transfusion-service-center_DDN+BO_0.4\n",
      "36 36\n",
      "X_train shape: (7427, 37) y_train shape: (7427,)\n",
      "X_test shape: (180, 37) y_test shape: (180,)\n",
      "Threshold 0.45 MAE: 0.05743015074806345 MSE: 0.006070267181582953\n",
      "Time: 34.757237911224365\n",
      "========================================\n",
      "8 car\n",
      "X_train shape: (2121, 37) y_train shape: (2121,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.05 MAE: 0.12029059418884591 MSE: 0.0322368965354076\n",
      "Time: 1.6707892417907715\n",
      "X_train shape: (3613, 37) y_train shape: (3613,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.1 MAE: 0.12073571061060787 MSE: 0.0322830593258502\n",
      "Time: 5.45780086517334\n",
      "X_train shape: (5658, 37) y_train shape: (5658,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.15000000000000002 MAE: 0.12214886831430505 MSE: 0.03234479091427819\n",
      "Time: 17.331878185272217\n",
      "X_train shape: (6078, 37) y_train shape: (6078,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.2 MAE: 0.13123663316952552 MSE: 0.03357628664126056\n",
      "Time: 21.167780876159668\n",
      "X_train shape: (6931, 37) y_train shape: (6931,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.25 MAE: 0.14301008990929026 MSE: 0.036699336321125146\n",
      "Time: 30.05872416496277\n",
      "X_train shape: (7117, 37) y_train shape: (7117,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.3 MAE: 0.14472099629243412 MSE: 0.037284560935398135\n",
      "Time: 29.42113995552063\n",
      "X_train shape: (7333, 37) y_train shape: (7333,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.35000000000000003 MAE: 0.1470986056149129 MSE: 0.03815680358143761\n",
      "Time: 31.453104972839355\n",
      "car_DDN+BO_0.35 car_DDN+BO_0.4\n",
      "36 36\n",
      "car_DDN+BO_0.4 car_DDN+BO_0.45\n",
      "36 36\n",
      "========================================\n",
      "9 christine\n",
      "X_train shape: (852, 37) y_train shape: (852,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.1764196703071066 MSE: 0.03438883217003362\n",
      "Time: 0.2689940929412842\n",
      "X_train shape: (2547, 37) y_train shape: (2547,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.11091442785805807 MSE: 0.014159515475815997\n",
      "Time: 2.4249022006988525\n",
      "X_train shape: (3581, 37) y_train shape: (3581,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.07924260356021232 MSE: 0.007392359977746454\n",
      "Time: 5.190668106079102\n",
      "X_train shape: (4927, 37) y_train shape: (4927,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.05573348629105769 MSE: 0.004094067473827435\n",
      "Time: 12.953248023986816\n",
      "X_train shape: (6180, 37) y_train shape: (6180,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.04053678228074689 MSE: 0.0033872588214387297\n",
      "Time: 22.976458072662354\n",
      "X_train shape: (6913, 37) y_train shape: (6913,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.043376004541245426 MSE: 0.004213053071000689\n",
      "Time: 27.523027896881104\n",
      "X_train shape: (7391, 37) y_train shape: (7391,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.04689458558085808 MSE: 0.004959379996288431\n",
      "Time: 31.324786901474\n",
      "christine_DDN+BO_0.35 christine_DDN+BO_0.4\n",
      "37 37\n",
      "X_train shape: (7595, 37) y_train shape: (7595,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.04226169789619459 MSE: 0.003964550503919295\n",
      "Time: 37.661861181259155\n",
      "========================================\n",
      "10 cnae-9\n",
      "X_train shape: (1549, 37) y_train shape: (1549,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n",
      "Threshold 0.05 MAE: 0.09440407667980756 MSE: 0.03652556357464156\n",
      "Time: 0.7698071002960205\n",
      "X_train shape: (4238, 37) y_train shape: (4238,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n",
      "Threshold 0.1 MAE: 0.10203963758060816 MSE: 0.03353062587306748\n",
      "Time: 9.354120016098022\n",
      "cnae-9_DDN+BO_0.1 cnae-9_DDN+BO_0.15\n",
      "20 20\n",
      "X_train shape: (5296, 37) y_train shape: (5296,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n",
      "Threshold 0.2 MAE: 0.11487076576571756 MSE: 0.03311196629183563\n",
      "Time: 15.470361948013306\n",
      "X_train shape: (6148, 37) y_train shape: (6148,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n",
      "Threshold 0.25 MAE: 0.1378992997297221 MSE: 0.034959468991857465\n",
      "Time: 22.28431797027588\n",
      "X_train shape: (6785, 37) y_train shape: (6785,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold 0.3 MAE: 0.15458539146685732 MSE: 0.03781551838298862\n",
      "Time: 25.67836284637451\n",
      "X_train shape: (7006, 37) y_train shape: (7006,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n",
      "Threshold 0.35000000000000003 MAE: 0.15737755731790035 MSE: 0.03840560698360381\n",
      "Time: 29.450445890426636\n",
      "X_train shape: (7187, 37) y_train shape: (7187,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n",
      "Threshold 0.4 MAE: 0.16013217959792223 MSE: 0.03901959589634166\n",
      "Time: 32.73251795768738\n",
      "X_train shape: (7403, 37) y_train shape: (7403,)\n",
      "X_test shape: (196, 37) y_test shape: (196,)\n",
      "Threshold 0.45 MAE: 0.16405120281703867 MSE: 0.039950277175857495\n",
      "Time: 33.27726984024048\n",
      "========================================\n",
      "11 connect-4\n",
      "X_train shape: (1694, 37) y_train shape: (1694,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.10822262010332873 MSE: 0.028072571685747862\n",
      "Time: 0.8819999694824219\n",
      "X_train shape: (2547, 37) y_train shape: (2547,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.13061193997923332 MSE: 0.029012824353409464\n",
      "Time: 2.2471840381622314\n",
      "X_train shape: (4530, 37) y_train shape: (4530,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.08645173987604512 MSE: 0.030744978197135134\n",
      "Time: 9.256585121154785\n",
      "X_train shape: (5336, 37) y_train shape: (5336,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.0901202235469374 MSE: 0.03342644508514349\n",
      "Time: 13.80149507522583\n",
      "X_train shape: (6612, 37) y_train shape: (6612,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.10304622595675977 MSE: 0.036757521496324386\n",
      "Time: 24.623175144195557\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.11593784052304633 MSE: 0.040508718305601196\n",
      "Time: 31.685606956481934\n",
      "X_train shape: (7391, 37) y_train shape: (7391,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.11556909588591024 MSE: 0.04039209611680325\n",
      "Time: 31.898640871047974\n",
      "connect-4_DDN+BO_0.35 connect-4_DDN+BO_0.4\n",
      "37 37\n",
      "X_train shape: (7595, 37) y_train shape: (7595,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.1051777512871752 MSE: 0.03733143634132063\n",
      "Time: 33.751670837402344\n",
      "========================================\n",
      "12 covertype\n",
      "X_train shape: (1694, 37) y_train shape: (1694,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.10911265634324681 MSE: 0.025429511800015007\n",
      "Time: 0.8628749847412109\n",
      "X_train shape: (1899, 37) y_train shape: (1899,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.11517545916025276 MSE: 0.024837969720125225\n",
      "Time: 1.222343921661377\n",
      "X_train shape: (4525, 37) y_train shape: (4525,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.10478343783779434 MSE: 0.03009649821940179\n",
      "Time: 9.490044116973877\n",
      "X_train shape: (5532, 37) y_train shape: (5532,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.11273694400155997 MSE: 0.03441643705134857\n",
      "Time: 15.21578073501587\n",
      "X_train shape: (6672, 37) y_train shape: (6672,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.1218057726242055 MSE: 0.03797076578133895\n",
      "Time: 24.651334762573242\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.13372448156778874 MSE: 0.04224309617737326\n",
      "Time: 31.56846785545349\n",
      "X_train shape: (7391, 37) y_train shape: (7391,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.13335529387696043 MSE: 0.04210486232834936\n",
      "Time: 31.62071704864502\n",
      "covertype_DDN+BO_0.35 covertype_DDN+BO_0.4\n",
      "37 37\n",
      "X_train shape: (7595, 37) y_train shape: (7595,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.12315740009698047 MSE: 0.03843622026046207\n",
      "Time: 33.683581829071045\n",
      "========================================\n",
      "13 credit-g\n",
      "X_train shape: (1284, 37) y_train shape: (1284,)\n",
      "X_test shape: (410, 37) y_test shape: (410,)\n",
      "Threshold 0.05 MAE: 0.1337560275155678 MSE: 0.019503998852051473\n",
      "Time: 0.4620687961578369\n",
      "X_train shape: (2564, 37) y_train shape: (2564,)\n",
      "X_test shape: (410, 37) y_test shape: (410,)\n",
      "Threshold 0.1 MAE: 0.07020895805359356 MSE: 0.006246849181335231\n",
      "Time: 2.2610390186309814\n",
      "X_train shape: (4331, 37) y_train shape: (4331,)\n",
      "X_test shape: (410, 37) y_test shape: (410,)\n",
      "Threshold 0.15000000000000002 MAE: 0.06028544744338289 MSE: 0.004905935440177674\n",
      "Time: 8.719321012496948\n",
      "X_train shape: (5770, 37) y_train shape: (5770,)\n",
      "X_test shape: (410, 37) y_test shape: (410,)\n",
      "Threshold 0.2 MAE: 0.03770064751013317 MSE: 0.002827843977108497\n",
      "Time: 17.549026012420654\n",
      "X_train shape: (6923, 37) y_train shape: (6923,)\n",
      "X_test shape: (410, 37) y_test shape: (410,)\n",
      "Threshold 0.25 MAE: 0.034527276881259845 MSE: 0.002949397731785822\n",
      "Time: 28.324529886245728\n",
      "X_train shape: (7189, 37) y_train shape: (7189,)\n",
      "X_test shape: (410, 37) y_test shape: (410,)\n",
      "Threshold 0.3 MAE: 0.035564183997491346 MSE: 0.003074988083352202\n",
      "Time: 28.64499592781067\n",
      "credit-g_DDN+BO_0.3 credit-g_DDN+BO_0.35\n",
      "36 36\n",
      "X_train shape: (7197, 37) y_train shape: (7197,)\n",
      "X_test shape: (410, 37) y_test shape: (410,)\n",
      "Threshold 0.4 MAE: 0.03540740508096726 MSE: 0.003056427156723003\n",
      "Time: 29.553295135498047\n",
      "credit-g_DDN+BO_0.4 credit-g_DDN+BO_0.45\n",
      "37 37\n",
      "========================================\n",
      "14 dilbert\n",
      "X_train shape: (1547, 37) y_train shape: (1547,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.13576912253505397 MSE: 0.040406441364191525\n",
      "Time: 0.6813290119171143\n",
      "X_train shape: (4002, 37) y_train shape: (4002,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.11544605116137066 MSE: 0.031282474316131785\n",
      "Time: 6.015737056732178\n",
      "X_train shape: (4218, 37) y_train shape: (4218,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.11459883023295726 MSE: 0.030794062248276247\n",
      "Time: 7.6113059520721436\n",
      "X_train shape: (5060, 37) y_train shape: (5060,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.11102236933406723 MSE: 0.027887906662630954\n",
      "Time: 11.783324003219604\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.11164249419556833 MSE: 0.023984266125913485\n",
      "Time: 19.82958173751831\n",
      "X_train shape: (6145, 37) y_train shape: (6145,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.111728506550602 MSE: 0.023970377970475926\n",
      "Time: 20.021878957748413\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.12165740940785301 MSE: 0.02376185428176376\n",
      "Time: 29.845887899398804\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.12330644312180823 MSE: 0.023874713416155078\n",
      "Time: 31.4798526763916\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.125592532404563 MSE: 0.024090656386687895\n",
      "Time: 42.291088819503784\n",
      "========================================\n",
      "15 dionis\n",
      "X_train shape: (1490, 37) y_train shape: (1490,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.05 MAE: 0.48772419378622983 MSE: 0.25294661173902994\n",
      "Time: 1.0278091430664062\n",
      "X_train shape: (2559, 37) y_train shape: (2559,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.1 MAE: 0.4474392579855105 MSE: 0.2152736121212263\n",
      "Time: 3.7452123165130615\n",
      "X_train shape: (3809, 37) y_train shape: (3809,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.15000000000000002 MAE: 0.4745492696604865 MSE: 0.24026873186990316\n",
      "Time: 8.799839973449707\n",
      "X_train shape: (5348, 37) y_train shape: (5348,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.2 MAE: 0.5182457464367511 MSE: 0.28365037623438755\n",
      "Time: 18.747596740722656\n",
      "X_train shape: (6408, 37) y_train shape: (6408,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.25 MAE: 0.5436102001590397 MSE: 0.31058377225155326\n",
      "Time: 25.28855323791504\n",
      "X_train shape: (7129, 37) y_train shape: (7129,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.3 MAE: 0.5629420883592712 MSE: 0.3319755173808996\n",
      "Time: 30.55572485923767\n",
      "X_train shape: (7403, 37) y_train shape: (7403,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold 0.35000000000000003 MAE: 0.5657887344123221 MSE: 0.3351886145224994\n",
      "Time: 29.722817182540894\n",
      "dionis_DDN+BO_0.35 dionis_DDN+BO_0.4\n",
      "37 37\n",
      "X_train shape: (7607, 37) y_train shape: (7607,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.45 MAE: 0.550723018384964 MSE: 0.3183675655136474\n",
      "Time: 32.79226088523865\n",
      "========================================\n",
      "16 fabert\n",
      "X_train shape: (665, 37) y_train shape: (665,)\n",
      "X_test shape: (188, 37) y_test shape: (188,)\n",
      "Threshold 0.05 MAE: 0.13307824901861848 MSE: 0.02605595348205156\n",
      "Time: 0.15297794342041016\n",
      "X_train shape: (1919, 37) y_train shape: (1919,)\n",
      "X_test shape: (188, 37) y_test shape: (188,)\n",
      "Threshold 0.1 MAE: 0.13195503230441924 MSE: 0.02575826367065046\n",
      "Time: 1.229133129119873\n",
      "X_train shape: (2977, 37) y_train shape: (2977,)\n",
      "X_test shape: (188, 37) y_test shape: (188,)\n",
      "Threshold 0.15000000000000002 MAE: 0.17732499323631876 MSE: 0.03979028634645062\n",
      "Time: 2.9963388442993164\n",
      "fabert_DDN+BO_0.15 fabert_DDN+BO_0.2\n",
      "15 15\n",
      "X_train shape: (4963, 37) y_train shape: (4963,)\n",
      "X_test shape: (188, 37) y_test shape: (188,)\n",
      "Threshold 0.25 MAE: 0.24889782754159323 MSE: 0.07029626167511484\n",
      "Time: 11.200273036956787\n",
      "X_train shape: (5784, 37) y_train shape: (5784,)\n",
      "X_test shape: (188, 37) y_test shape: (188,)\n",
      "Threshold 0.3 MAE: 0.2768472728775515 MSE: 0.08499054561992757\n",
      "Time: 16.26504898071289\n",
      "X_train shape: (6912, 37) y_train shape: (6912,)\n",
      "X_test shape: (188, 37) y_test shape: (188,)\n",
      "Threshold 0.35000000000000003 MAE: 0.2831253799658214 MSE: 0.08850611390098086\n",
      "Time: 26.020636320114136\n",
      "X_train shape: (7623, 37) y_train shape: (7623,)\n",
      "X_test shape: (188, 37) y_test shape: (188,)\n",
      "Threshold 0.4 MAE: 0.30100379782130954 MSE: 0.09894941942304193\n",
      "Time: 34.26459217071533\n",
      "fabert_DDN+BO_0.4 fabert_DDN+BO_0.45\n",
      "38 38\n",
      "========================================\n",
      "17 fashion_mnist\n",
      "X_train shape: (1762, 37) y_train shape: (1762,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.1738551688280984 MSE: 0.06674011878391865\n",
      "Time: 0.9596071243286133\n",
      "X_train shape: (3857, 37) y_train shape: (3857,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.1752739530372527 MSE: 0.06748234743928505\n",
      "Time: 5.794028043746948\n",
      "X_train shape: (5442, 37) y_train shape: (5442,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.1609273588117168 MSE: 0.057746448711831294\n",
      "Time: 14.489039182662964\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.1630364099038475 MSE: 0.05958218984371572\n",
      "Time: 20.80732011795044\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.15475452152470998 MSE: 0.05218520259610852\n",
      "Time: 28.87659215927124\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.1536313388453664 MSE: 0.05127855252433879\n",
      "Time: 32.285593032836914\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.15208111602042731 MSE: 0.050088847317708136\n",
      "Time: 33.64609098434448\n",
      "fashion_mnist_DDN+BO_0.35 fashion_mnist_DDN+BO_0.4\n",
      "36 36\n",
      "fashion_mnist_DDN+BO_0.4 fashion_mnist_DDN+BO_0.45\n",
      "36 36\n",
      "========================================\n",
      "18 guillermo\n",
      "X_train shape: (836, 37) y_train shape: (836,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.05 MAE: 0.1103259244625343 MSE: 0.014890264625813861\n",
      "Time: 0.23235082626342773\n",
      "X_train shape: (2090, 37) y_train shape: (2090,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.1 MAE: 0.09243983140603251 MSE: 0.01126357744767675\n",
      "Time: 1.4281961917877197\n",
      "X_train shape: (3148, 37) y_train shape: (3148,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.15000000000000002 MAE: 0.05651587464597693 MSE: 0.004772962264585768\n",
      "Time: 3.7398788928985596\n",
      "X_train shape: (3544, 37) y_train shape: (3544,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.2 MAE: 0.04546346393243567 MSE: 0.003334605270705083\n",
      "Time: 5.006444692611694\n",
      "X_train shape: (5134, 37) y_train shape: (5134,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.25 MAE: 0.050517758612586415 MSE: 0.0035041901573011395\n",
      "Time: 12.871515989303589\n",
      "X_train shape: (6171, 37) y_train shape: (6171,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.3 MAE: 0.06646370337524535 MSE: 0.006146657883034118\n",
      "Time: 20.113847017288208\n",
      "X_train shape: (6897, 37) y_train shape: (6897,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.35000000000000003 MAE: 0.08033848397995975 MSE: 0.009104577929019272\n",
      "Time: 26.116657972335815\n",
      "X_train shape: (7794, 37) y_train shape: (7794,)\n",
      "X_test shape: (17, 37) y_test shape: (17,)\n",
      "Threshold 0.4 MAE: 0.08206249727939419 MSE: 0.009452708477031615\n",
      "Time: 34.80766725540161\n",
      "guillermo_DDN+BO_0.4 guillermo_DDN+BO_0.45\n",
      "38 38\n",
      "========================================\n",
      "19 helena\n",
      "X_train shape: (8, 37) y_train shape: (8,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.15000000000000002 MAE: 0.07749804213272951 MSE: 0.009583776794550924\n",
      "Time: 0.021183013916015625\n",
      "helena_DDN+BO_0.15 helena_DDN+BO_0.2\n",
      "1 1\n",
      "X_train shape: (224, 37) y_train shape: (224,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.25 MAE: 0.3476022857923587 MSE: 0.12466826220766805\n",
      "Time: 0.10247611999511719\n",
      "X_train shape: (410, 37) y_train shape: (410,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.3 MAE: 0.34624326289826995 MSE: 0.12372481405814378\n",
      "Time: 0.25460004806518555\n",
      "X_train shape: (1246, 37) y_train shape: (1246,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.35000000000000003 MAE: 0.3482644444121642 MSE: 0.12504412518808017\n",
      "Time: 0.4686129093170166\n",
      "X_train shape: (1263, 37) y_train shape: (1263,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.4 MAE: 0.34963994278508026 MSE: 0.12600409153712103\n",
      "Time: 0.48299503326416016\n",
      "X_train shape: (2331, 37) y_train shape: (2331,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.45 MAE: 0.3570814353885901 MSE: 0.13126315344554265\n",
      "Time: 1.7972102165222168\n",
      "========================================\n",
      "20 higgs\n",
      "X_train shape: (642, 37) y_train shape: (642,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.11298671020049521 MSE: 0.014336947877918345\n",
      "Time: 0.15511703491210938\n",
      "X_train shape: (1675, 37) y_train shape: (1675,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.08529691315803836 MSE: 0.008823271344552887\n",
      "Time: 0.8897888660430908\n",
      "X_train shape: (2949, 37) y_train shape: (2949,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.03567698724416159 MSE: 0.002126250571788895\n",
      "Time: 3.079545259475708\n",
      "higgs_DDN+BO_0.15 higgs_DDN+BO_0.2\n",
      "15 15\n",
      "X_train shape: (4935, 37) y_train shape: (4935,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.0581474361703584 MSE: 0.004411858020981744\n",
      "Time: 11.817447185516357\n",
      "X_train shape: (5560, 37) y_train shape: (5560,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.07819542136572284 MSE: 0.007685475118550596\n",
      "Time: 14.68503212928772\n",
      "X_train shape: (6824, 37) y_train shape: (6824,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.08855558082270792 MSE: 0.00941304209083481\n",
      "Time: 25.724426984786987\n",
      "X_train shape: (7595, 37) y_train shape: (7595,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.10799700353752378 MSE: 0.01323430396907158\n",
      "Time: 33.08598017692566\n",
      "higgs_DDN+BO_0.4 higgs_DDN+BO_0.45\n",
      "38 38\n",
      "========================================\n",
      "21 jannis\n",
      "X_train shape: (637, 37) y_train shape: (637,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.12259887444961698 MSE: 0.020297787674642676\n",
      "Time: 0.14282703399658203\n",
      "X_train shape: (1675, 37) y_train shape: (1675,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.1225988513524064 MSE: 0.020297781972465304\n",
      "Time: 0.8124771118164062\n",
      "X_train shape: (2949, 37) y_train shape: (2949,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.12259879173218596 MSE: 0.020297767253590275\n",
      "Time: 2.9833078384399414\n",
      "jannis_DDN+BO_0.15 jannis_DDN+BO_0.2\n",
      "15 15\n",
      "X_train shape: (4935, 37) y_train shape: (4935,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold 0.25 MAE: 0.12259871632249092 MSE: 0.020297748636663823\n",
      "Time: 11.797086954116821\n",
      "X_train shape: (5756, 37) y_train shape: (5756,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.12259868768675349 MSE: 0.020297741567159185\n",
      "Time: 16.650169849395752\n",
      "X_train shape: (6884, 37) y_train shape: (6884,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.122598680665346 MSE: 0.020297739833735503\n",
      "Time: 25.26802897453308\n",
      "X_train shape: (7595, 37) y_train shape: (7595,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.12259866251879455 MSE: 0.020297735353770642\n",
      "Time: 33.578943967819214\n",
      "jannis_DDN+BO_0.4 jannis_DDN+BO_0.45\n",
      "38 38\n",
      "========================================\n",
      "22 jasmine\n",
      "X_train shape: (1274, 37) y_train shape: (1274,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.10467971613439785 MSE: 0.012257571739545448\n",
      "Time: 0.4640839099884033\n",
      "X_train shape: (3240, 37) y_train shape: (3240,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.09237404164327383 MSE: 0.009513351158924486\n",
      "Time: 3.804222822189331\n",
      "X_train shape: (4934, 37) y_train shape: (4934,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.08942577282888546 MSE: 0.008945246505392159\n",
      "Time: 11.371541976928711\n",
      "X_train shape: (6396, 37) y_train shape: (6396,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.06920407471247814 MSE: 0.005995701165184591\n",
      "Time: 21.868853092193604\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.058026287851865775 MSE: 0.0051854857676080675\n",
      "Time: 28.78311324119568\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.06148095606054223 MSE: 0.005365414808179102\n",
      "Time: 32.18139696121216\n",
      "jasmine_DDN+BO_0.3 jasmine_DDN+BO_0.35\n",
      "36 36\n",
      "X_train shape: (7391, 37) y_train shape: (7391,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.061789111692722186 MSE: 0.005384652873284413\n",
      "Time: 31.653523921966553\n",
      "jasmine_DDN+BO_0.4 jasmine_DDN+BO_0.45\n",
      "37 37\n",
      "========================================\n",
      "23 jungle_chess_2pcs_raw_endgame_complete\n",
      "X_train shape: (1478, 37) y_train shape: (1478,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.09915243161115382 MSE: 0.01505932320767044\n",
      "Time: 0.6326568126678467\n",
      "X_train shape: (2758, 37) y_train shape: (2758,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.07285101449031901 MSE: 0.007283775335837865\n",
      "Time: 2.5737128257751465\n",
      "X_train shape: (4525, 37) y_train shape: (4525,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.06962174988981075 MSE: 0.006567365068065407\n",
      "Time: 9.263343095779419\n",
      "X_train shape: (5964, 37) y_train shape: (5964,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.06702802163145012 MSE: 0.0056964334981806685\n",
      "Time: 18.878693103790283\n",
      "X_train shape: (7117, 37) y_train shape: (7117,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.06979535357198097 MSE: 0.006388983013687708\n",
      "Time: 27.788721323013306\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.07072034592158956 MSE: 0.006616750115787866\n",
      "Time: 30.4361310005188\n",
      "jungle_chess_2pcs_raw_endgame_complete_DDN+BO_0.3 jungle_chess_2pcs_raw_endgame_complete_DDN+BO_0.35\n",
      "36 36\n",
      "X_train shape: (7391, 37) y_train shape: (7391,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.07058996415237541 MSE: 0.006584702499278948\n",
      "Time: 31.677685976028442\n",
      "jungle_chess_2pcs_raw_endgame_complete_DDN+BO_0.4 jungle_chess_2pcs_raw_endgame_complete_DDN+BO_0.45\n",
      "37 37\n",
      "========================================\n",
      "24 kc1\n",
      "X_train shape: (2171, 37) y_train shape: (2171,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.07616102992323508 MSE: 0.007238342219550085\n",
      "Time: 1.494002103805542\n",
      "X_train shape: (3663, 37) y_train shape: (3663,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.057612333885429495 MSE: 0.005401846814364325\n",
      "Time: 4.954305171966553\n",
      "X_train shape: (5708, 37) y_train shape: (5708,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.07832603574678385 MSE: 0.00750829125580804\n",
      "Time: 16.225467920303345\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.09976884534348301 MSE: 0.010809480488235322\n",
      "Time: 20.525840997695923\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.12639957035393404 MSE: 0.016444492216410894\n",
      "Time: 26.819108247756958\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.1301771316440164 MSE: 0.01738176505811701\n",
      "Time: 28.7948579788208\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.1354342737685773 MSE: 0.018743829840119228\n",
      "Time: 30.751437187194824\n",
      "kc1_DDN+BO_0.35 kc1_DDN+BO_0.4\n",
      "36 36\n",
      "kc1_DDN+BO_0.4 kc1_DDN+BO_0.45\n",
      "36 36\n",
      "========================================\n",
      "25 KDDCup09_appetency\n",
      "X_train shape: (1196, 37) y_train shape: (1196,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.05 MAE: 0.10569332298136658 MSE: 0.011182711175904697\n",
      "Time: 0.38323092460632324\n",
      "X_train shape: (2449, 37) y_train shape: (2449,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.1 MAE: 0.12136070332129945 MSE: 0.014740052963701688\n",
      "Time: 2.058962821960449\n",
      "X_train shape: (3579, 37) y_train shape: (3579,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.15000000000000002 MAE: 0.1450493081880584 MSE: 0.021050934458895564\n",
      "Time: 4.827533960342407\n",
      "X_train shape: (4427, 37) y_train shape: (4427,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.2 MAE: 0.16266675875373146 MSE: 0.026472107056505906\n",
      "Time: 8.898945093154907\n",
      "X_train shape: (5269, 37) y_train shape: (5269,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.25 MAE: 0.1805975161190625 MSE: 0.03262709548143626\n",
      "Time: 13.57492995262146\n",
      "X_train shape: (6337, 37) y_train shape: (6337,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.3 MAE: 0.22512976924782566 MSE: 0.050695045654640454\n",
      "Time: 21.850197076797485\n",
      "KDDCup09_appetency_DDN+BO_0.3 KDDCup09_appetency_DDN+BO_0.35\n",
      "28 28\n",
      "X_train shape: (7190, 37) y_train shape: (7190,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.4 MAE: 0.2539886573886746 MSE: 0.06452187073516276\n",
      "Time: 28.580389976501465\n",
      "X_train shape: (7376, 37) y_train shape: (7376,)\n",
      "X_test shape: (7, 37) y_test shape: (7,)\n",
      "Threshold 0.45 MAE: 0.2581111157106445 MSE: 0.06663298070645493\n",
      "Time: 30.422779083251953\n",
      "========================================\n",
      "26 kr-vs-kp\n",
      "X_train shape: (505, 37) y_train shape: (505,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.05 MAE: 0.14997934191915432 MSE: 0.044357447423700895\n",
      "Time: 0.07823920249938965\n",
      "X_train shape: (1781, 37) y_train shape: (1781,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.1 MAE: 0.16385041085463675 MSE: 0.0390698656895566\n",
      "Time: 1.045140266418457\n",
      "X_train shape: (3104, 37) y_train shape: (3104,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.15000000000000002 MAE: 0.16750619092218877 MSE: 0.03828928631502431\n",
      "Time: 3.442840099334717\n",
      "X_train shape: (4168, 37) y_train shape: (4168,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.2 MAE: 0.17844885306953118 MSE: 0.037481072611248785\n",
      "Time: 7.627270221710205\n",
      "X_train shape: (4384, 37) y_train shape: (4384,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.25 MAE: 0.17977358286108097 MSE: 0.03753869919484646\n",
      "Time: 8.548532962799072\n",
      "X_train shape: (5658, 37) y_train shape: (5658,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.3 MAE: 0.19171719646502347 MSE: 0.03957432241841854\n",
      "Time: 16.08022403717041\n",
      "X_train shape: (6078, 37) y_train shape: (6078,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.35000000000000003 MAE: 0.19949993367015847 MSE: 0.042369523906860475\n",
      "Time: 19.177930116653442\n",
      "X_train shape: (6931, 37) y_train shape: (6931,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold 0.4 MAE: 0.2091932101716409 MSE: 0.04739652631257709\n",
      "Time: 26.538382053375244\n",
      "X_train shape: (7117, 37) y_train shape: (7117,)\n",
      "X_test shape: (266, 37) y_test shape: (266,)\n",
      "Threshold 0.45 MAE: 0.21079166197431087 MSE: 0.04824804617304786\n",
      "Time: 28.686530828475952\n",
      "========================================\n",
      "27 mfeat-factors\n",
      "X_train shape: (1203, 37) y_train shape: (1203,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.09253647709430125 MSE: 0.022385090950761372\n",
      "Time: 0.40268492698669434\n",
      "X_train shape: (2240, 37) y_train shape: (2240,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.10621330987899275 MSE: 0.022985196439836028\n",
      "Time: 1.631654977798462\n",
      "X_train shape: (3822, 37) y_train shape: (3822,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.1307252522097194 MSE: 0.026126365665042863\n",
      "Time: 5.399312973022461\n",
      "X_train shape: (4218, 37) y_train shape: (4218,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.1351041756507667 MSE: 0.026875521538675125\n",
      "Time: 8.03940486907959\n",
      "X_train shape: (5060, 37) y_train shape: (5060,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.14877972269867643 MSE: 0.029641186031185998\n",
      "Time: 12.146276950836182\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.18405564501857238 MSE: 0.03945953895689721\n",
      "Time: 20.610377073287964\n",
      "mfeat-factors_DDN+BO_0.3 mfeat-factors_DDN+BO_0.35\n",
      "28 28\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.20677422151987127 MSE: 0.04794649711349835\n",
      "Time: 26.867738962173462\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.20998220918208443 MSE: 0.049282390744517506\n",
      "Time: 28.29530119895935\n",
      "========================================\n",
      "28 MiniBooNE\n",
      "X_train shape: (2171, 37) y_train shape: (2171,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.08024086771419424 MSE: 0.008785297896504788\n",
      "Time: 1.5352840423583984\n",
      "X_train shape: (3447, 37) y_train shape: (3447,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.06223168187708591 MSE: 0.006743470326019052\n",
      "Time: 4.233083248138428\n",
      "X_train shape: (5708, 37) y_train shape: (5708,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.08220967771250454 MSE: 0.009056874195873617\n",
      "Time: 15.772439956665039\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.10163458727735188 MSE: 0.012308695231756745\n",
      "Time: 19.69994592666626\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.1260385368801909 MSE: 0.017883466561180752\n",
      "Time: 26.81466794013977\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.12979317598820703 MSE: 0.01881229930522655\n",
      "Time: 28.149733781814575\n",
      "MiniBooNE_DDN+BO_0.3 MiniBooNE_DDN+BO_0.35\n",
      "35 35\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.13505112190819746 MSE: 0.02016258399356774\n",
      "Time: 31.26298189163208\n",
      "MiniBooNE_DDN+BO_0.4 MiniBooNE_DDN+BO_0.45\n",
      "36 36\n",
      "========================================\n",
      "29 nomao\n",
      "X_train shape: (1565, 37) y_train shape: (1565,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.08102892266749065 MSE: 0.011962541204851629\n",
      "Time: 0.7304399013519287\n",
      "X_train shape: (3154, 37) y_train shape: (3154,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.09011273133753193 MSE: 0.012857595095327379\n",
      "Time: 3.65933895111084\n",
      "X_train shape: (4218, 37) y_train shape: (4218,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.11462075396500147 MSE: 0.016563467395522505\n",
      "Time: 8.340128898620605\n",
      "X_train shape: (4434, 37) y_train shape: (4434,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.11764709722897276 MSE: 0.017151712102433086\n",
      "Time: 9.077849864959717\n",
      "X_train shape: (5708, 37) y_train shape: (5708,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.14559292060651002 MSE: 0.024014469530235956\n",
      "Time: 16.262419939041138\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.1640525868036995 MSE: 0.029963738442044897\n",
      "Time: 20.37253499031067\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.1875462868467942 MSE: 0.03882300845851911\n",
      "Time: 26.459555864334106\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.1912546948968884 MSE: 0.04021080009306614\n",
      "Time: 28.985043048858643\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.19646845984921307 MSE: 0.042203146602693656\n",
      "Time: 30.6825110912323\n",
      "========================================\n",
      "30 numerai28.6\n",
      "X_train shape: (186, 37) y_train shape: (186,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.05611412177302184 MSE: 0.0037670210842483255\n",
      "Time: 0.09573769569396973\n",
      "X_train shape: (1047, 37) y_train shape: (1047,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.014455210518725328 MSE: 0.0002960367038447005\n",
      "Time: 0.33811306953430176\n",
      "X_train shape: (1467, 37) y_train shape: (1467,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.028408172336512518 MSE: 0.0010839145074395802\n",
      "Time: 0.6605749130249023\n",
      "X_train shape: (2945, 37) y_train shape: (2945,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.020184900281469784 MSE: 0.0005858491942780926\n",
      "Time: 3.0999789237976074\n",
      "X_train shape: (3161, 37) y_train shape: (3161,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.030573125237441667 MSE: 0.0012259136891162546\n",
      "Time: 3.516752243041992\n",
      "X_train shape: (4707, 37) y_train shape: (4707,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.09487034302722905 MSE: 0.009291579688436066\n",
      "Time: 10.391041040420532\n",
      "X_train shape: (5548, 37) y_train shape: (5548,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.13050901329422887 MSE: 0.017323800253365173\n",
      "Time: 14.923552989959717\n",
      "X_train shape: (6824, 37) y_train shape: (6824,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.16682269905465627 MSE: 0.028121010622212374\n",
      "Time: 25.210549116134644\n",
      "========================================\n",
      "31 phoneme\n",
      "X_train shape: (1762, 37) y_train shape: (1762,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.044779761979912444 MSE: 0.002585330080571844\n",
      "Time: 0.9192638397216797\n",
      "X_train shape: (3425, 37) y_train shape: (3425,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.04521534751354635 MSE: 0.002582715421874368\n",
      "Time: 4.2624499797821045\n",
      "X_train shape: (5435, 37) y_train shape: (5435,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.04834525642735991 MSE: 0.0035028444522404167\n",
      "Time: 13.970103025436401\n",
      "X_train shape: (6145, 37) y_train shape: (6145,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.04498898363842105 MSE: 0.002795402976815784\n",
      "Time: 20.20959997177124\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.053767949551222466 MSE: 0.00452645066167272\n",
      "Time: 26.433613777160645\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.05589165727795153 MSE: 0.004916622872777791\n",
      "Time: 28.784791946411133\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.059189848019772194 MSE: 0.005517095464172822\n",
      "Time: 30.973907947540283\n",
      "phoneme_DDN+BO_0.35 phoneme_DDN+BO_0.4\n",
      "36 36\n",
      "X_train shape: (7391, 37) y_train shape: (7391,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.05949112426192504 MSE: 0.005573660687059435\n",
      "Time: 31.98475694656372\n",
      "========================================\n",
      "32 riccardo\n",
      "X_train shape: (1771, 37) y_train shape: (1771,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold 0.05 MAE: 0.06397672097092386 MSE: 0.005507088697262857\n",
      "Time: 0.9649078845977783\n",
      "X_train shape: (2396, 37) y_train shape: (2396,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n",
      "Threshold 0.1 MAE: 0.0664500373564995 MSE: 0.005918461383618992\n",
      "Time: 1.8074049949645996\n",
      "X_train shape: (4374, 37) y_train shape: (4374,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n",
      "Threshold 0.15000000000000002 MAE: 0.045128334440335226 MSE: 0.0030802362309454755\n",
      "Time: 8.438350915908813\n",
      "riccardo_DDN+BO_0.15 riccardo_DDN+BO_0.2\n",
      "20 20\n",
      "X_train shape: (5432, 37) y_train shape: (5432,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n",
      "Threshold 0.25 MAE: 0.04328820048833907 MSE: 0.003146078888153113\n",
      "Time: 13.990693092346191\n",
      "X_train shape: (6284, 37) y_train shape: (6284,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n",
      "Threshold 0.3 MAE: 0.05723476651637711 MSE: 0.00554221838354652\n",
      "Time: 21.06878113746643\n",
      "X_train shape: (6921, 37) y_train shape: (6921,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n",
      "Threshold 0.35000000000000003 MAE: 0.07962204086392949 MSE: 0.008723354812327926\n",
      "Time: 25.734996795654297\n",
      "X_train shape: (7142, 37) y_train shape: (7142,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n",
      "Threshold 0.4 MAE: 0.08333000020805414 MSE: 0.009367236799769342\n",
      "Time: 27.561407804489136\n",
      "X_train shape: (7323, 37) y_train shape: (7323,)\n",
      "X_test shape: (60, 37) y_test shape: (60,)\n",
      "Threshold 0.45 MAE: 0.0870809936978832 MSE: 0.010032740431435743\n",
      "Time: 30.383518934249878\n",
      "========================================\n",
      "33 robert\n",
      "X_train shape: (420, 37) y_train shape: (420,)\n",
      "X_test shape: (8, 37) y_test shape: (8,)\n",
      "Threshold 0.15000000000000002 MAE: 0.07289167302190476 MSE: 0.008518376134482452\n",
      "Time: 0.3566257953643799\n",
      "X_train shape: (606, 37) y_train shape: (606,)\n",
      "X_test shape: (8, 37) y_test shape: (8,)\n",
      "Threshold 0.2 MAE: 0.07570321479598231 MSE: 0.008731618273820376\n",
      "Time: 0.5522029399871826\n",
      "X_train shape: (1459, 37) y_train shape: (1459,)\n",
      "X_test shape: (8, 37) y_test shape: (8,)\n",
      "Threshold 0.25 MAE: 0.23613047661728265 MSE: 0.058607914487505075\n",
      "Time: 0.5980818271636963\n",
      "robert_DDN+BO_0.25 robert_DDN+BO_0.3\n",
      "9 9\n",
      "X_train shape: (2527, 37) y_train shape: (2527,)\n",
      "X_test shape: (8, 37) y_test shape: (8,)\n",
      "Threshold 0.35000000000000003 MAE: 0.26299939570622366 MSE: 0.07201899464183883\n",
      "Time: 2.1006038188934326\n",
      "X_train shape: (3369, 37) y_train shape: (3369,)\n",
      "X_test shape: (8, 37) y_test shape: (8,)\n",
      "Threshold 0.4 MAE: 0.314650036494398 MSE: 0.101854957965926\n",
      "Time: 4.001616716384888\n",
      "X_train shape: (3765, 37) y_train shape: (3765,)\n",
      "X_test shape: (8, 37) y_test shape: (8,)\n",
      "Threshold 0.45 MAE: 0.3365069034969999 MSE: 0.11608720860113922\n",
      "Time: 5.181474924087524\n",
      "========================================\n",
      "34 segment\n",
      "X_train shape: (1565, 37) y_train shape: (1565,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.09681799250143529 MSE: 0.01567126795531917\n",
      "Time: 0.7333130836486816\n",
      "X_train shape: (3154, 37) y_train shape: (3154,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.10212493278100582 MSE: 0.015672231200507973\n",
      "Time: 3.5398459434509277\n",
      "X_train shape: (4218, 37) y_train shape: (4218,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.11457662023829995 MSE: 0.017237091767623877\n",
      "Time: 8.090038061141968\n",
      "X_train shape: (4434, 37) y_train shape: (4434,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.1162038963933912 MSE: 0.017572587210953804\n",
      "Time: 9.257384061813354\n",
      "X_train shape: (5912, 37) y_train shape: (5912,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.14514936789173172 MSE: 0.025930988165006365\n",
      "Time: 17.64039897918701\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.14665945145748577 MSE: 0.02650012051429361\n",
      "Time: 19.49226403236389\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.16529129050026256 MSE: 0.033477383174426036\n",
      "Time: 27.141132831573486\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.16825681047264232 MSE: 0.03460249947970913\n",
      "Time: 29.220943212509155\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.17244422272878193 MSE: 0.0362270034203087\n",
      "Time: 32.299452781677246\n",
      "========================================\n",
      "35 shuttle\n",
      "X_train shape: (999, 37) y_train shape: (999,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.05 MAE: 0.09764884737678856 MSE: 0.011424926565074905\n",
      "Time: 0.2970428466796875\n",
      "X_train shape: (2252, 37) y_train shape: (2252,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.1 MAE: 0.10465270086720306 MSE: 0.013103123740696243\n",
      "Time: 1.6863000392913818\n",
      "X_train shape: (3166, 37) y_train shape: (3166,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.15000000000000002 MAE: 0.11161090177462774 MSE: 0.014844327276605873\n",
      "Time: 3.5212759971618652\n",
      "X_train shape: (4230, 37) y_train shape: (4230,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.2 MAE: 0.14074025286731387 MSE: 0.02253470618486699\n",
      "Time: 8.1284019947052\n",
      "X_train shape: (5072, 37) y_train shape: (5072,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.25 MAE: 0.15782999513308607 MSE: 0.027742265665612855\n",
      "Time: 12.399442911148071\n",
      "X_train shape: (5924, 37) y_train shape: (5924,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.3 MAE: 0.1994700055371055 MSE: 0.04278573961374142\n",
      "Time: 18.7015597820282\n",
      "X_train shape: (6140, 37) y_train shape: (6140,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.35000000000000003 MAE: 0.20209573977419823 MSE: 0.04384804963066381\n",
      "Time: 20.041454792022705\n",
      "X_train shape: (6993, 37) y_train shape: (6993,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.4 MAE: 0.23072359124101524 MSE: 0.05634272976254764\n",
      "Time: 26.60773205757141\n",
      "X_train shape: (7179, 37) y_train shape: (7179,)\n",
      "X_test shape: (204, 37) y_test shape: (204,)\n",
      "Threshold 0.45 MAE: 0.234756338615541 MSE: 0.05823716302483761\n",
      "Time: 28.16458225250244\n",
      "========================================\n",
      "36 sylvine\n",
      "X_train shape: (1469, 37) y_train shape: (1469,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.06750626522666428 MSE: 0.00732885764484508\n",
      "Time: 0.5202741622924805\n",
      "X_train shape: (4218, 37) y_train shape: (4218,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.09330552678953598 MSE: 0.010405404504531991\n",
      "Time: 6.8625500202178955\n",
      "sylvine_DDN+BO_0.1 sylvine_DDN+BO_0.15\n",
      "20 20\n",
      "X_train shape: (5492, 37) y_train shape: (5492,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.2 MAE: 0.12082462124604146 MSE: 0.015851017286004264\n",
      "Time: 12.339342832565308\n",
      "X_train shape: (6128, 37) y_train shape: (6128,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.14592759728090238 MSE: 0.022579099538202505\n",
      "Time: 16.10554599761963\n",
      "X_train shape: (6981, 37) y_train shape: (6981,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.17088391490602373 MSE: 0.030879088646042212\n",
      "Time: 22.48576807975769\n",
      "X_train shape: (6986, 37) y_train shape: (6986,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.17096573388253594 MSE: 0.030908543878565054\n",
      "Time: 22.19336700439453\n",
      "X_train shape: (7167, 37) y_train shape: (7167,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.17451830495918455 MSE: 0.03218890752028921\n",
      "Time: 23.97838592529297\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.17963308689304672 MSE: 0.034071957464599865\n",
      "Time: 25.842049598693848\n",
      "========================================\n",
      "37 vehicle\n",
      "X_train shape: (1694, 37) y_train shape: (1694,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.1294578619401986 MSE: 0.020826171599665502\n",
      "Time: 0.6718130111694336\n",
      "X_train shape: (2547, 37) y_train shape: (2547,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.13161471588783236 MSE: 0.02246719385523017\n",
      "Time: 1.8241429328918457\n",
      "X_train shape: (4530, 37) y_train shape: (4530,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.12784706646560404 MSE: 0.02246319119328378\n",
      "Time: 7.8726348876953125\n",
      "X_train shape: (5120, 37) y_train shape: (5120,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold 0.2 MAE: 0.12777449435250854 MSE: 0.023731225436217137\n",
      "Time: 11.055739164352417\n",
      "X_train shape: (6612, 37) y_train shape: (6612,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.1290658180347148 MSE: 0.027642695978602035\n",
      "Time: 20.2863130569458\n",
      "X_train shape: (7383, 37) y_train shape: (7383,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.1332570044112673 MSE: 0.03102732902512062\n",
      "Time: 25.721511125564575\n",
      "X_train shape: (7391, 37) y_train shape: (7391,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.13308609549828765 MSE: 0.030921414809125327\n",
      "Time: 25.773870944976807\n",
      "vehicle_DDN+BO_0.35 vehicle_DDN+BO_0.4\n",
      "37 37\n",
      "X_train shape: (7595, 37) y_train shape: (7595,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.45 MAE: 0.12951930245741733 MSE: 0.028162057429211445\n",
      "Time: 28.160654067993164\n",
      "========================================\n",
      "38 volkert\n",
      "X_train shape: (637, 37) y_train shape: (637,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.05 MAE: 0.12259887860961427 MSE: 0.02029780179792185\n",
      "Time: 0.10846686363220215\n",
      "X_train shape: (1675, 37) y_train shape: (1675,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.1 MAE: 0.12259887624627043 MSE: 0.020297801214465786\n",
      "Time: 0.7031629085540771\n",
      "X_train shape: (2949, 37) y_train shape: (2949,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.15000000000000002 MAE: 0.12259892911647814 MSE: 0.02029781426692511\n",
      "Time: 2.492506980895996\n",
      "volkert_DDN+BO_0.15 volkert_DDN+BO_0.2\n",
      "15 15\n",
      "X_train shape: (4935, 37) y_train shape: (4935,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.25 MAE: 0.12259900095474387 MSE: 0.020297832002178363\n",
      "Time: 9.680837869644165\n",
      "X_train shape: (5756, 37) y_train shape: (5756,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.3 MAE: 0.12259902883406658 MSE: 0.020297838884959393\n",
      "Time: 13.216780185699463\n",
      "X_train shape: (6884, 37) y_train shape: (6884,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.35000000000000003 MAE: 0.12259903511044676 MSE: 0.020297840434457454\n",
      "Time: 22.1890869140625\n",
      "X_train shape: (7595, 37) y_train shape: (7595,)\n",
      "X_test shape: (216, 37) y_test shape: (216,)\n",
      "Threshold 0.4 MAE: 0.12259905290110007 MSE: 0.020297844826572774\n",
      "Time: 27.542495012283325\n",
      "volkert_DDN+BO_0.4 volkert_DDN+BO_0.45\n",
      "38 38\n"
     ]
    }
   ],
   "source": [
    "target_test_pred_time = {}\n",
    "data_names = list(DDN.keys())\n",
    "data_embed = DDN\n",
    "\n",
    "for data_index, data_name in enumerate(data_names):\n",
    "    print(\"=\"*40)\n",
    "    print(data_index, data_name)\n",
    "    \n",
    "    task_test = [data_name]\n",
    "    candidate_score = DDN[data_name]\n",
    "    prev_key = None\n",
    "    dict_key = None\n",
    "    len_task_train = 0\n",
    "    prev_len_task_train = 0\n",
    "    for threshold in np.arange(0.05, 0.5, 0.05):\n",
    "        prev_key = dict_key\n",
    "        dict_key = '_'.join([data_name, combination_name, str(round(threshold,2))])\n",
    "        \n",
    "        start = time.time()\n",
    "        task_train = [i for i in data_names if i!=data_name and abs(DDN[i]-candidate_score)<threshold]\n",
    "        prev_len_task_train = len_task_train\n",
    "        len_task_train = len(task_train)\n",
    "        if len_task_train == 0:\n",
    "            target_test_pred_time[dict_key] = 'N.A.'\n",
    "            pickle.dump(target_test_pred_time, open(log_target_test_pred_time, \"wb\"))\n",
    "            continue\n",
    "            \n",
    "        if len_task_train == prev_len_task_train:\n",
    "            print(prev_key, dict_key)\n",
    "            print(prev_len_task_train, len_task_train)\n",
    "            target_test_pred_time[dict_key] =  target_test_pred_time[prev_key]\n",
    "            continue\n",
    "\n",
    "        X_train = np.array([np.hstack((data_embed[i[0]], i[1])) for i in pipeline_steps_class if i[0] in task_train and i[3] != \"Timeout\" and not np.isinf(i[3])])\n",
    "        y_train = np.array([i[3] for i in pipeline_steps_class if i[0] in task_train and i[3] != \"Timeout\" and not np.isinf(i[3])])\n",
    "        X_test = np.array([np.hstack((data_embed[i[0]], i[1])) for i in pipeline_steps_class if i[0] in task_test and i[3] != \"Timeout\" and not np.isinf(i[3])])\n",
    "        y_test = np.array([i[3] for i in pipeline_steps_class if i[0] in task_test and i[3] != \"Timeout\" and not np.isinf(i[3])])\n",
    "        print(\"X_train shape:\", X_train.shape, \"y_train shape:\", y_train.shape)\n",
    "        print(\"X_test shape:\", X_test.shape, \"y_test shape:\", y_test.shape)\n",
    "\n",
    "        model = GaussianProcessRegressor(\n",
    "            kernel=Matern(nu=2.5),alpha=1e-6,normalize_y=True,\n",
    "            n_restarts_optimizer=0,random_state=1\n",
    "        )\n",
    "        \n",
    "        import warnings\n",
    "        with warnings.catch_warnings():\n",
    "            warnings.simplefilter(\"ignore\")\n",
    "            model.fit(X_train, y_train)\n",
    "        end = time.time()\n",
    "        duration = end-start\n",
    "\n",
    "        y_pred = model.predict(X_test)\n",
    "        mae = mean_absolute_error(y_test, y_pred)\n",
    "        mse = mean_squared_error(y_test, y_pred)\n",
    "        print(\"Threshold\", threshold, \"MAE:\", mae, \"MSE:\", mse)\n",
    "        print(\"Time:\", duration)\n",
    "        target_test_pred_time[dict_key] = (y_test, y_pred, duration, mae, mse)\n",
    "        pickle.dump(target_test_pred_time, open(log_target_test_pred_time, 'wb'))\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "25c82606",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d129480",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}