add support for multiple objectives in botorch to numerical (#88)

* add support for multiple objectives in botorch `to_numerical`

---------

Signed-off-by: Grossberger Lukas (CR/AIR2.2) <Lukas.Grossberger@de.bosch.com>

Author: LGro

blackboxopt/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "4.10.0"
+__version__ = "4.11.0"
 
 from parameterspace import ParameterSpace
 

blackboxopt/optimizers/botorch_base.py

@@ -294,7 +294,7 @@ def _append_evaluations_to_data(
         X, Y = to_numerical(
             _evals,
             self.search_space,
-            self.objective,
+            objectives=[self.objective],
             batch_shape=self.batch_shape,
             torch_dtype=self.torch_dtype,
         )

blackboxopt/optimizers/botorch_utils.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import warnings
-from typing import Iterable, List, Optional, Tuple
+from typing import Iterable, List, Optional, Sequence, Tuple
 
 import numpy as np
 import parameterspace as ps
@@ -13,6 +13,7 @@
 
 from blackboxopt.base import ConstraintsError, Objective
 from blackboxopt.evaluation import Evaluation
+from blackboxopt.utils import get_loss_vector
 
 
 def filter_y_nans(
@@ -88,7 +89,7 @@ def impute_nans_with_constant(x: torch.Tensor, c: float = -1.0) -> torch.Tensor:
 def to_numerical(
     evaluations: Iterable[Evaluation],
     search_space: ps.ParameterSpace,
-    objective: Objective,
+    objectives: Sequence[Objective],
     constraint_names: Optional[List[str]] = None,
     batch_shape: torch.Size = torch.Size(),
     torch_dtype: torch.dtype = torch.float32,
@@ -102,7 +103,7 @@ def to_numerical(
     Args:
         evaluations: List of evaluations that were collected during optimization.
         search_space: Search space used during optimization.
-        objective: Objective that was used for optimization.
+        objectives: Objectives that were used for optimization.
         constraint_names: Name of constraints that are used for optimization.
         batch_shape: Batch dimension(s) used for batched models.
         torch_dtype: Type of returned tensors.
@@ -142,13 +143,15 @@ def to_numerical(
         dtype=torch_dtype,
     )
     X = X.reshape(*batch_shape + X.shape)
-    Y = torch.tensor(
-        np.array([[e.objectives[objective.name]] for e in evaluations], dtype=float),
-        dtype=torch_dtype,
-    )
 
-    if objective.greater_is_better:
-        Y *= -1
+    Y = torch.Tensor(
+        [
+            get_loss_vector(
+                known_objectives=objectives, reported_objectives=e.objectives
+            )
+            for e in evaluations
+        ]
+    ).to(dtype=torch_dtype)
 
     if constraint_names is not None:
         try:
@@ -163,12 +166,12 @@ def to_numerical(
         except KeyError as e:
             raise ConstraintsError(
                 f"Constraint name {e} is not defined in input evaluations."
-            )
-        except TypeError:
+            ) from e
+        except TypeError as e:
             raise ConstraintsError(
                 f"Constraint name(s) {constraint_names} are not defined in input "
                 + "evaluations."
-            )
+            ) from e
 
     Y = Y.reshape(*batch_shape + Y.shape)
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "blackboxopt"
-version = "4.10.0"
+version = "4.11.0"
 description = "A common interface for blackbox optimization algorithms along with useful helpers like parallel optimization loops, analysis and visualization scripts."
 readme = "README.md"
 repository = "https://github.com/boschresearch/blackboxopt"

tests/optimizers/botorch_utils_test.py

@@ -54,7 +54,7 @@ def test_to_numerical(evaluations, search_space, greater_is_better):
 
     objective = Objective(objective_name, greater_is_better)
 
-    X, Y = to_numerical(evaluations, search_space, objective)
+    X, Y = to_numerical(evaluations, search_space, [objective])
 
     assert X.dtype == torch.float32
     assert Y.dtype == torch.float32
@@ -82,7 +82,7 @@ def test_to_numerical_with_batch(evaluations, search_space):
     objective = Objective(objective_name, False)
 
     batch_shape = torch.Size((1,))
-    X, Y = to_numerical(evaluations, search_space, objective, batch_shape=batch_shape)
+    X, Y = to_numerical(evaluations, search_space, [objective], batch_shape=batch_shape)
 
     assert X.dtype == torch.float32
     assert Y.dtype == torch.float32
@@ -102,31 +102,31 @@ def test_to_numerical_raises_errors(search_space):
         objectives={objective_name: 0.64},
     )
     with pytest.raises(ValueError, match="Mismatch"):
-        to_numerical([eval_err], search_space, objective)
+        to_numerical([eval_err], search_space, [objective])
 
     # parameter not within defined bounds -> invalid configuration
     eval_err = Evaluation(
         configuration={"x0": 0.1, "x1": False, "x2": "small", "fp": 0.5},
         objectives={objective_name: 0.64},
     )
     with pytest.raises(ValueError, match="not valid"):
-        to_numerical([eval_err], search_space, objective)
+        to_numerical([eval_err], search_space, [objective])
 
     # conditional parameter should be active, but is inactive -> invalid configuration
     eval_err = Evaluation(
         configuration={"x0": 0.57, "x1": True, "x2": "small", "fp": 0.5},
         objectives={objective_name: 0.64},
     )
     with pytest.raises(ValueError, match="not valid"):
-        to_numerical([eval_err], search_space, objective)
+        to_numerical([eval_err], search_space, [objective])
 
     # conditional parameter should be inactive, but is active -> invalid configuration
     eval_err = Evaluation(
         configuration={"x0": 0.57, "x1": False, "x2": "small", "cp": 0.3, "fp": 0.5},
         objectives={objective_name: 0.64},
     )
     with pytest.raises(ValueError, match="not valid"):
-        to_numerical([eval_err], search_space, objective)
+        to_numerical([eval_err], search_space, [objective])
 
 
 @pytest.mark.parametrize(
@@ -147,7 +147,7 @@ def test_to_numerical_with_constraints(
     _, Y = to_numerical(
         evaluations_with_constraints,
         search_space,
-        objective,
+        [objective],
         constraints,
     )
 
@@ -172,7 +172,7 @@ def test_to_numerical_raises_error_on_wrong_constraints(
         to_numerical(
             evaluations_with_constraints,
             search_space,
-            objective,
+            [objective],
             constraint_names=["WRONG_NAME"],
         )
 
@@ -190,11 +190,32 @@ def test_to_numerical_raises_error_on_wrong_constraints(
         to_numerical(
             evaluations,
             search_space,
-            objective,
+            [objective],
             constraint_names=[constraint_name_1],
         )
 
 
+def test_to_numerical_multiple_objectives(search_space):
+    objectives = [
+        Objective("score", greater_is_better=True),
+        Objective("loss", greater_is_better=False),
+    ]
+
+    evaluations = [
+        Evaluation(
+            objectives={"score": 0.1, "loss": 0.1}, configuration=search_space.sample()
+        )
+    ]
+
+    X, Y = to_numerical(evaluations, search_space, objectives)
+
+    assert X.dtype == torch.float32
+    assert Y.dtype == torch.float32
+    assert X.size() == (len(evaluations), len(search_space))
+    assert Y.size() == (len(evaluations), len(objectives))
+    assert torch.equal(Y, torch.Tensor([[-0.1, 0.1]]))
+
+
 def test_filter_y_nans():
     x1 = torch.tensor([[0.1], [0.7], [1.0]])
     y1 = torch.tensor([[0.8], [0.3], [0.5]])