XGBoost Regressor .ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Import necessary libraries\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, mean_squared_error\nfrom sklearn.datasets import load_breast_cancer, fetch_california_housing\nfrom xgboost import XGBClassifier, XGBRegressor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T10:46:28.988785Z","iopub.execute_input":"2025-01-02T10:46:28.989074Z","iopub.status.idle":"2025-01-02T10:46:31.245792Z","shell.execute_reply.started":"2025-01-02T10:46:28.989045Z","shell.execute_reply":"2025-01-02T10:46:31.244667Z"}},"outputs":[],"execution_count":1},{"cell_type":"code","source":"# Regression Task: California Housing Dataset\n# Load dataset\ndata_reg = fetch_california_housing(as_frame=True)\nX_reg = data_reg.data\ny_reg = data_reg.target","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T10:46:31.247120Z","iopub.execute_input":"2025-01-02T10:46:31.247618Z","iopub.status.idle":"2025-01-02T10:46:33.982890Z","shell.execute_reply.started":"2025-01-02T10:46:31.247584Z","shell.execute_reply":"2025-01-02T10:46:33.982161Z"}},"outputs":[],"execution_count":2},{"cell_type":"code","source":"data_reg","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T10:46:33.985169Z","iopub.execute_input":"2025-01-02T10:46:33.985432Z","iopub.status.idle":"2025-01-02T10:46:34.077673Z","shell.execute_reply.started":"2025-01-02T10:46:33.985409Z","shell.execute_reply":"2025-01-02T10:46:34.076734Z"}},"outputs":[{"execution_count":3,"output_type":"execute_result","data":{"text/plain":"{'data':        MedInc  HouseAge  AveRooms  AveBedrms  Population  AveOccup  Latitude  \\\n 0      8.3252      41.0  6.984127   1.023810       322.0  2.555556     37.88   \n 1      8.3014      21.0  6.238137   0.971880      2401.0  2.109842     37.86   \n 2      7.2574      52.0  8.288136   1.073446       496.0  2.802260     37.85   \n 3      5.6431      52.0  5.817352   1.073059       558.0  2.547945     37.85   \n 4      3.8462      52.0  6.281853   1.081081       565.0  2.181467     37.85   \n ...       ...       ...       ...        ...         ...       ...       ...   \n 20635  1.5603      25.0  5.045455   1.133333       845.0  2.560606     39.48   \n 20636  2.5568      18.0  6.114035   1.315789       356.0  3.122807     39.49   \n 20637  1.7000      17.0  5.205543   1.120092      1007.0  2.325635     39.43   \n 20638  1.8672      18.0  5.329513   1.171920       741.0  2.123209     39.43   \n 20639  2.3886      16.0  5.254717   1.162264      1387.0  2.616981     39.37   \n \n        Longitude  \n 0        -122.23  \n 1        -122.22  \n 2        -122.24  \n 3        -122.25  \n 4        -122.25  \n ...          ...  \n 20635    -121.09  \n 20636    -121.21  \n 20637    -121.22  \n 20638    -121.32  \n 20639    -121.24  \n \n [20640 rows x 8 columns],\n 'target': 0        4.526\n 1        3.585\n 2        3.521\n 3        3.413\n 4        3.422\n          ...  \n 20635    0.781\n 20636    0.771\n 20637    0.923\n 20638    0.847\n 20639    0.894\n Name: MedHouseVal, Length: 20640, dtype: float64,\n 'frame':        MedInc  HouseAge  AveRooms  AveBedrms  Population  AveOccup  Latitude  \\\n 0      8.3252      41.0  6.984127   1.023810       322.0  2.555556     37.88   \n 1      8.3014      21.0  6.238137   0.971880      2401.0  2.109842     37.86   \n 2      7.2574      52.0  8.288136   1.073446       496.0  2.802260     37.85   \n 3      5.6431      52.0  5.817352   1.073059       558.0  2.547945     37.85   \n 4      3.8462      52.0  6.281853   1.081081       565.0  2.181467     37.85   \n ...       ...       ...       ...        ...         ...       ...       ...   \n 20635  1.5603      25.0  5.045455   1.133333       845.0  2.560606     39.48   \n 20636  2.5568      18.0  6.114035   1.315789       356.0  3.122807     39.49   \n 20637  1.7000      17.0  5.205543   1.120092      1007.0  2.325635     39.43   \n 20638  1.8672      18.0  5.329513   1.171920       741.0  2.123209     39.43   \n 20639  2.3886      16.0  5.254717   1.162264      1387.0  2.616981     39.37   \n \n        Longitude  MedHouseVal  \n 0        -122.23        4.526  \n 1        -122.22        3.585  \n 2        -122.24        3.521  \n 3        -122.25        3.413  \n 4        -122.25        3.422  \n ...          ...          ...  \n 20635    -121.09        0.781  \n 20636    -121.21        0.771  \n 20637    -121.22        0.923  \n 20638    -121.32        0.847  \n 20639    -121.24        0.894  \n \n [20640 rows x 9 columns],\n 'target_names': ['MedHouseVal'],\n 'feature_names': ['MedInc',\n  'HouseAge',\n  'AveRooms',\n  'AveBedrms',\n  'Population',\n  'AveOccup',\n  'Latitude',\n  'Longitude'],\n 'DESCR': '.. _california_housing_dataset:\\n\\nCalifornia Housing dataset\\n--------------------------\\n\\n**Data Set Characteristics:**\\n\\n    :Number of Instances: 20640\\n\\n    :Number of Attributes: 8 numeric, predictive attributes and the target\\n\\n    :Attribute Information:\\n        - MedInc        median income in block group\\n        - HouseAge      median house age in block group\\n        - AveRooms      average number of rooms per household\\n        - AveBedrms     average number of bedrooms per household\\n        - Population    block group population\\n        - AveOccup      average number of household members\\n        - Latitude      block group latitude\\n        - Longitude     block group longitude\\n\\n    :Missing Attribute Values: None\\n\\nThis dataset was obtained from the StatLib repository.\\nhttps://www.dcc.fc.up.pt/~ltorgo/Regression/cal_housing.html\\n\\nThe target variable is the median house value for California districts,\\nexpressed in hundreds of thousands of dollars ($100,000).\\n\\nThis dataset was derived from the 1990 U.S. census, using one row per census\\nblock group. A block group is the smallest geographical unit for which the U.S.\\nCensus Bureau publishes sample data (a block group typically has a population\\nof 600 to 3,000 people).\\n\\nA household is a group of people residing within a home. Since the average\\nnumber of rooms and bedrooms in this dataset are provided per household, these\\ncolumns may take surprisingly large values for block groups with few households\\nand many empty houses, such as vacation resorts.\\n\\nIt can be downloaded/loaded using the\\n:func:`sklearn.datasets.fetch_california_housing` function.\\n\\n.. topic:: References\\n\\n    - Pace, R. Kelley and Ronald Barry, Sparse Spatial Autoregressions,\\n      Statistics and Probability Letters, 33 (1997) 291-297\\n'}"},"metadata":{}}],"execution_count":3},{"cell_type":"code","source":"# Split the dataset into train and test sets\nX_train_reg, X_test_reg, y_train_reg, y_test_reg = train_test_split(\n    X_reg, y_reg, test_size=0.2, random_state=42\n)\n\n# Initialize the XGBoost regressor\nxgb_reg = XGBRegressor(\n    n_estimators=100,\n    learning_rate=0.1,\n    max_depth=3,\n    random_state=42\n)\n\n# Fit the model\nxgb_reg.fit(X_train_reg, y_train_reg)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-02T10:46:34.079063Z","iopub.execute_input":"2025-01-02T10:46:34.079311Z","iopub.status.idle":"2025-01-02T10:46:34.231925Z","shell.execute_reply.started":"2025-01-02T10:46:34.079290Z","shell.execute_reply":"2025-01-02T10:46:34.230339Z"}},"outputs":[{"execution_count":4,"output_type":"execute_result","data":{"text/plain":"XGBRegressor(base_score=None, booster=None, callbacks=None,\n             colsample_bylevel=None, colsample_bynode=None,\n             colsample_bytree=None, device=None, early_stopping_rounds=None,\n             enable_categorical=False, eval_metric=None, feature_types=None,\n             gamma=None, grow_policy=None, importance_type=None,\n             interaction_constraints=None, learning_rate=0.1, max_bin=None,\n             max_cat_threshold=None, max_cat_to_onehot=None,\n             max_delta_step=None, max_depth=3, max_leaves=None,\n             min_child_weight=None, missing=nan, monotone_constraints=None,\n             multi_strategy=None, n_estimators=100, n_jobs=None,\n             num_parallel_tree=None, random_state=42, ...)","text/html":"<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>XGBRegressor(base_score=None, booster=None, callbacks=None,\n             colsample_bylevel=None, colsample_bynode=None,\n             colsample_bytree=None, device=None, early_stopping_rounds=None,\n             enable_categorical=False, eval_metric=None, feature_types=None,\n             gamma=None, grow_policy=None, importance_type=None,\n             interaction_constraints=None, learning_rate=0.1, max_bin=None,\n             max_cat_threshold=None, max_cat_to_onehot=None,\n             max_delta_step=None, max_depth=3, max_leaves=None,\n             min_child_weight=None, missing=nan, monotone_constraints=None,\n             multi_strategy=None, n_estimators=100, n_jobs=None,\n             num_parallel_tree=None, random_state=42, ...)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">XGBRegressor</label><div class=\"sk-toggleable__content\"><pre>XGBRegressor(base_score=None, booster=None, callbacks=None,\n             colsample_bylevel=None, colsample_bynode=None,\n             colsample_bytree=None, device=None, early_stopping_rounds=None,\n             enable_categorical=False, eval_metric=None, feature_types=None,\n             gamma=None, grow_policy=None, importance_type=None,\n             interaction_constraints=None, learning_rate=0.1, max_bin=None,\n             max_cat_threshold=None, max_cat_to_onehot=None,\n             max_delta_step=None, max_depth=3, max_leaves=None,\n             min_child_weight=None, missing=nan, monotone_constraints=None,\n             multi_strategy=None, n_estimators=100, n_jobs=None,\n             num_parallel_tree=None, random_state=42, ...)</pre></div></div></div></div></div>"},"metadata":{}}],"execution_count":4},{"cell_type":"code","source":"# Make predictions\ny_pred_reg = xgb_reg.predict(X_test_reg)\n\n# Evaluate the model\nmse = mean_squared_error(y_test_reg, y_pred_reg)\nprint(f\"Regression Mean Squared Error: {mse:.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T10:46:34.232468Z","iopub.execute_input":"2025-01-02T10:46:34.232714Z","iopub.status.idle":"2025-01-02T10:46:34.243670Z","shell.execute_reply.started":"2025-01-02T10:46:34.232692Z","shell.execute_reply":"2025-01-02T10:46:34.242956Z"}},"outputs":[{"name":"stdout","text":"Regression Mean Squared Error: 0.30\n","output_type":"stream"}],"execution_count":5},{"cell_type":"code","source":"# Make predictions\ny_pred_reg = xgb_reg.predict(X_test_reg)\ny_pred_reg","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T10:46:34.244346Z","iopub.execute_input":"2025-01-02T10:46:34.244564Z","iopub.status.idle":"2025-01-02T10:46:34.263300Z","shell.execute_reply.started":"2025-01-02T10:46:34.244543Z","shell.execute_reply":"2025-01-02T10:46:34.262213Z"}},"outputs":[{"execution_count":6,"output_type":"execute_result","data":{"text/plain":"array([0.59980536, 1.0359062 , 3.9847162 , ..., 4.6957827 , 0.91971457,\n       1.8906444 ], dtype=float32)"},"metadata":{}}],"execution_count":6},{"cell_type":"code","source":"y_test_reg","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T10:46:34.264036Z","iopub.execute_input":"2025-01-02T10:46:34.264272Z","iopub.status.idle":"2025-01-02T10:46:34.277768Z","shell.execute_reply.started":"2025-01-02T10:46:34.264251Z","shell.execute_reply":"2025-01-02T10:46:34.276971Z"}},"outputs":[{"execution_count":7,"output_type":"execute_result","data":{"text/plain":"20046    0.47700\n3024     0.45800\n15663    5.00001\n20484    2.18600\n9814     2.78000\n          ...   \n15362    2.63300\n16623    2.66800\n18086    5.00001\n2144     0.72300\n3665     1.51500\nName: MedHouseVal, Length: 4128, dtype: float64"},"metadata":{}}],"execution_count":7},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}