diff --git a/mlos_bench/mlos_bench/tests/tunables/test_tunables_size_props.py b/mlos_bench/mlos_bench/tests/tunables/test_tunables_size_props.py new file mode 100644 index 00000000000..de966536c4b --- /dev/null +++ b/mlos_bench/mlos_bench/tests/tunables/test_tunables_size_props.py @@ -0,0 +1,97 @@ +# +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +""" +Unit tests for checking tunable size properties. +""" + +import numpy as np +import pytest + +from mlos_bench.tunables.tunable import Tunable + + +# Note: these test do *not* check the ConfigSpace conversions for those same Tunables. +# That is checked indirectly via grid_search_optimizer_test.py + +def test_tunable_int_size_props() -> None: + """Test tunable int size properties""" + tunable = Tunable( + name="test", + config={ + "type": "int", + "range": [1, 5], + "default": 3, + }) + assert tunable.span == 4 + assert tunable.cardinality == 5 + expected = [1, 2, 3, 4, 5] + assert list(tunable.quantized_values or []) == expected + assert list(tunable.values or []) == expected + + +def test_tunable_float_size_props() -> None: + """Test tunable float size properties""" + tunable = Tunable( + name="test", + config={ + "type": "float", + "range": [1.5, 5], + "default": 3, + }) + assert tunable.span == 3.5 + assert tunable.cardinality == np.inf + assert tunable.quantized_values is None + assert tunable.values is None + + +def test_tunable_categorical_size_props() -> None: + """Test tunable categorical size properties""" + tunable = Tunable( + name="test", + config={ + "type": "categorical", + "values": ["a", "b", "c"], + "default": "a", + }) + with pytest.raises(AssertionError): + _ = tunable.span + assert tunable.cardinality == 3 + assert tunable.values == ["a", "b", "c"] + with pytest.raises(AssertionError): + _ = tunable.quantized_values + + +def test_tunable_quantized_int_size_props() -> None: + """Test quantized tunable int size properties""" + tunable = Tunable( + name="test", + config={ + "type": "int", + "range": [100, 1000], + "default": 100, + "quantization": 100 + }) + assert tunable.span == 900 + assert tunable.cardinality == 10 + expected = [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000] + assert list(tunable.quantized_values or []) == expected + assert list(tunable.values or []) == expected + + +def test_tunable_quantized_float_size_props() -> None: + """Test quantized tunable float size properties""" + tunable = Tunable( + name="test", + config={ + "type": "float", + "range": [0, 1], + "default": 0, + "quantization": .1 + }) + assert tunable.span == 1 + assert tunable.cardinality == 11 + expected = [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] + assert pytest.approx(list(tunable.quantized_values or []), 0.0001) == expected + assert pytest.approx(list(tunable.values or []), 0.0001) == expected diff --git a/mlos_bench/mlos_bench/tunables/tunable.py b/mlos_bench/mlos_bench/tunables/tunable.py index f04dbea8729..1e6f2023372 100644 --- a/mlos_bench/mlos_bench/tunables/tunable.py +++ b/mlos_bench/mlos_bench/tunables/tunable.py @@ -9,7 +9,9 @@ import collections import logging -from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple, Type, TypedDict, Union +from typing import Any, Dict, Iterable, List, Literal, Optional, Sequence, Tuple, Type, TypedDict, Union + +import numpy as np _LOG = logging.getLogger(__name__) @@ -370,6 +372,7 @@ def is_valid(self, value: TunableValue) -> bool: is_valid : bool True if the value is valid, False otherwise. """ + # FIXME: quantization check? if self.is_categorical and self._values: return value in self._values elif self.is_numerical and self._range: @@ -563,19 +566,79 @@ def range(self) -> Union[Tuple[int, int], Tuple[float, float]]: assert self._range is not None return self._range + @property + def span(self) -> Union[int, float]: + """ + Gets the span of the range. + + Note: this does not take quantization into account. + + Returns + ------- + Union[int, float] + (max - min) for numerical tunables. + """ + num_range = self.range + return num_range[1] - num_range[0] + @property def quantization(self) -> Optional[Union[int, float]]: """ - Get the number of quantization points, if specified. + Get the quantization factor, if specified. Returns ------- quantization : int, float, None - Number of quantization points or None. + The quantization factor, or None. """ - assert self.is_numerical + if self.is_categorical: + return None return self._quantization + @property + def quantized_values(self) -> Optional[Union[Iterable[int], Iterable[float]]]: + """ + Get a sequence of quanitized values for this tunable. + + Returns + ------- + Optional[Union[Iterable[int], Iterable[float]]] + If the Tunable is quantizable, returns a sequence of those elements, + else None (e.g., for unquantized float type tunables). + """ + num_range = self.range + if self.type == "float": + if not self._quantization: + return None + # Be sure to return python types instead of numpy types. + cardinality = self.cardinality + assert isinstance(cardinality, int) + return (float(x) for x in np.linspace(start=num_range[0], + stop=num_range[1], + num=cardinality, + endpoint=True)) + assert self.type == "int", f"Unhandled tunable type: {self}" + return range(int(num_range[0]), int(num_range[1]) + 1, int(self._quantization or 1)) + + @property + def cardinality(self) -> Union[int, float]: + """ + Gets the cardinality of elements in this tunable, or else infinity. + + If the tunable has quantization set, this + + Returns + ------- + cardinality : int, float + Either the number of points in the tunable or else infinity. + """ + if self.is_categorical: + return len(self.categories) + if not self.quantization and self.type == "float": + return np.inf + q_factor = self.quantization or 1 + return int(self.span / q_factor) + 1 + @property def is_log(self) -> Optional[bool]: """ @@ -629,6 +692,21 @@ def categories(self) -> List[Optional[str]]: assert self._values is not None return self._values + @property + def values(self) -> Optional[Union[Iterable[Optional[str]], Iterable[int], Iterable[float]]]: + """ + Gets the categories or quantized values for this tunable. + + Returns + ------- + Optional[Union[Iterable[Optional[str]], Iterable[int], Iterable[float]]] + Categories or quantized values. + """ + if self.is_categorical: + return self.categories + assert self.is_numerical + return self.quantized_values + @property def meta(self) -> Dict[str, Any]: """