Merge pull request #7 from mle-infrastructure/mle-search

`mle-search` & doc-strings

CHANGELOG.md

@@ -1,3 +1,22 @@
+## [v0.0.6] - [02/19/2022]
+### Added
+
+- Adds a command line interface for running a sequential search given a python script `<script>.py` containing a function `main(config)`, a default configuration file `<base>.yaml` & a search configuration `<search>.yaml`. The `main` function should return a single scalar performance score. You can then start the search via:
+
+  ```
+  mle-search <script>.py --base_config <base>.yaml --search_config <search>.yaml --num_iters <search_iters>
+  ```
+  Or short via:
+  ```
+  mle-search <script>.py -base <base>.yaml -search <search>.yaml -iters <search_iters>
+  ```
+- Adds doc-strings to all functionalities.
+### Changed
+
+- Make it possible to optimize parameters in nested dictionaries. Added helpers `flatten_config` and `unflatten_config`. For shaping `'sub1/sub2/vname' <-> {sub1: {sub2: {vname: v}}}`
+- Make start-up message also print fixed parameter settings.
+- Cleaned up decorator with the help of `Strategies` wrapper.
+
 ## [v0.0.5] - [01/05/2022]
 
 ### Added

README.md

@@ -221,19 +221,29 @@ strategy.refine(top_k=2)
 Note that the search space refinement is only implemented for random, SMBO and `nevergrad`-based search strategies.
 
 
+### Simple Command Line interface ⌨️
+
+You can also directly launch a search for your applications. This requires a couple of things: A python script `<script>.py` containing a function `main(config)`, which runs your simulation for a given configuration dictionary. It should return a single scalar performance score, which will be logged. Furthermore, you will need a search configuration `<search>.yaml` file and can add default fixed parameter settings in `<base>.yaml`.
+
+```
+mle-search <script>.py -base <base>.yaml -search <search>.yaml -iters <search_iters>
+```
+
+Have a look at the [example](https://github.com/mle-infrastructure/mle-hyperopt/tree/main/examples/mle_search), which can be executed via `mle-search run_mle_search.py -search search.yaml -base base.yaml`.
+
 ### Citing the MLE-Infrastructure ✏️
 
 If you use `mle-hyperopt` in your research, please cite it as follows:
 
 ```
 @software{mle_infrastructure2021github,
   author = {Robert Tjarko Lange},
-  title = {{MLE-Infrastructure}: A Set of Lightweight Tools
 for Distributed Machine Learning Experimentation},
+  title = {{MLE-Infrastructure}: A Set of Lightweight Tools for Distributed Machine Learning Experimentation},
   url = {http://github.com/mle-infrastructure},
   year = {2021},
 }
 ```
 
 ## Development 👷
 
-You can run the test suite via `python -m pytest -vv tests/`. If you find a bug or are missing your favourite feature, feel free to create an issue and/or start [contributing](CONTRIBUTING.md) :hugs:.
+You can run the test suite via `python -m pytest -vv tests/`. If you find a bug or are missing your favourite feature, feel free to create an issue and/or start [contributing](CONTRIBUTING.md) 🤗.

examples/mle_search/run_mle_search.py

@@ -0,0 +1,8 @@
+def main(config):
+    """Optimum: lrate=0.2, batch_size=4, arch='conv'."""
+    f1 = (
+        (config["lrate"] - 0.2) ** 2
+        + ((config["batch_size"] - 4) / 4) ** 2
+        + (0 if config["arch"]["sub_arch"] == "conv" else 0.2)
+    )
+    return f1

mle_hyperopt/__init__.py

@@ -8,20 +8,10 @@
     HalvingSearch,
     HyperbandSearch,
     PBTSearch,
+    Strategies,
 )
 from .decorator import hyperopt
 
-Strategies = {
-    "Random": RandomSearch,
-    "Grid": GridSearch,
-    "SMBO": SMBOSearch,
-    "Nevergrad": NevergradSearch,
-    "Coordinate": CoordinateSearch,
-    "Halving": HalvingSearch,
-    "Hyperband": HyperbandSearch,
-    "PBT": PBTSearch,
-}
-
 __all__ = [
     "__version__",
     "RandomSearch",

mle_hyperopt/_version.py

@@ -1 +1 @@
-__version__ = "0.0.5"
+__version__ = "0.0.6"

mle_hyperopt/cli.py

@@ -0,0 +1,131 @@
+import argparse
+import os
+import importlib
+from mle_logging import load_config
+from .strategies import Strategies
+
+
+def get_search_args() -> None:
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "exec_fname",
+        metavar="C",
+        type=str,
+        default="main.py",
+        help="Filename to import `main(config)` function from.",
+    )
+    parser.add_argument(
+        "-base",
+        "--base_config",
+        type=str,
+        default="base.yaml",
+        help="Filename to load base configuration from.",
+    )
+    parser.add_argument(
+        "-search",
+        "--search_config",
+        type=str,
+        default="search.yaml",
+        help="Filename to load search configuration from.",
+    )
+    parser.add_argument(
+        "-iters",
+        "--num_iters",
+        type=int,
+        default=None,
+        help="Number of desired search iterations.",
+    )
+    parser.add_argument(
+        "-log",
+        "--log_dir",
+        type=str,
+        default=None,
+        help="Directory to save search_log.yaml in.",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def search() -> None:
+    """Command line tool for running a sequential search given a python script
+    `<script>.py` containing a function `main(config)`, a default configuration
+    file `<base>.yaml` & a search configuration `<search>.yaml`. The `main`
+    function should return a single scalar performance score.
+    You can then start the search via:
+
+        mle-search <script>.py
+            --base_config <base>.yaml
+            --search_config <search>.yaml
+            --log_dir <log_dir>
+
+    Or short:
+
+        mle-search <script>.py -base <base>.yaml -search <search>.yaml -log <log_dir>
+
+    This will spawn single runs for different configurations and walk through a
+    set of search iterations.
+    """
+    args = get_search_args()
+
+    # Load base configuration and search configuration
+    search_config = load_config(args.search_config, True)
+    base_config = load_config(args.base_config, True)
+
+    # Setup search instance
+    real = (
+        search_config.search_config.real
+        if "real" in search_config.search_config.keys()
+        else None
+    )
+    integer = (
+        search_config.search_config.integer
+        if "integer" in search_config.search_config.keys()
+        else None
+    )
+    categorical = (
+        search_config.search_config.categorical
+        if "categorical" in search_config.search_config.keys()
+        else None
+    )
+
+    strategy = Strategies[search_config.search_type](
+        real,
+        integer,
+        categorical,
+        search_config.search_config,
+        search_config.maximize_objective,
+        fixed_params=base_config.toDict(),
+        verbose=search_config.verbose,
+    )
+
+    # Setup log storage path & effective search iterations
+    save_path = (
+        os.path.join(args.log_dir, "search_log.yaml")
+        if args.log_dir is not None
+        else "search_log.yaml"
+    )
+
+    num_search_iters = (
+        args.num_iters
+        if args.num_iters is not None
+        else search_config.num_iters
+    )
+
+    # Load the main function module
+    spec = importlib.util.spec_from_file_location(
+        "main", os.path.join(os.getcwd(), args.exec_fname)
+    )
+    foo = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(foo)
+
+    # Run the search loop and store results to path
+    for s_iter in range(num_search_iters):
+        config = strategy.ask()
+        # Add search id for logging inside main call
+        config["search_eval_id"] = (
+            search_config.search_type.lower() + f"_{s_iter}"
+        )
+        result = foo.main(config)
+        del config["search_eval_id"]
+        strategy.tell(config, result, save=True, save_path=save_path)

mle_hyperopt/decorator.py

@@ -1,29 +1,19 @@
-from typing import Union
+from typing import Optional, Callable, Any
 import functools
-from .strategies import (
-    RandomSearch,
-    GridSearch,
-    SMBOSearch,
-    NevergradSearch,
-    CoordinateSearch,
-    HalvingSearch,
-    HyperbandSearch,
-    PBTSearch,
-)
+from .strategies import Strategies
 
 
 def hyperopt(
     strategy_type: str,
     num_search_iters: int,
-    real: Union[dict, None] = None,
-    integer: Union[dict, None] = None,
-    categorical: Union[dict, None] = None,
-    search_config: Union[dict, None] = None,
+    real: Optional[dict] = None,
+    integer: Optional[dict] = None,
+    categorical: Optional[dict] = None,
+    search_config: Optional[dict] = None,
     maximize_objective: bool = False,
-    fixed_params: Union[dict, None] = None,
-):
-    """
-    Simple search decorator for all strategies. Example usage:
+    fixed_params: Optional[dict] = None,
+) -> Callable[[Any], None]:
+    """Simple search decorator for all strategies. Example usage:
     @hyperopt(strategy_type="grid",
               num_search_iters=25,
               real={"x": {"begin": 0., "end": 0.5, "bins": 5},
@@ -34,53 +24,48 @@ def distance_from_circle(config):
 
     strategy = distance_from_circle()
     strategy.log
+
+    Args:
+        strategy_type (str): Name of search strategy.
+        num_search_iters (int): Number of iterations to run.
+        real (Optional[dict], optional):
+                Dictionary of real-valued search variables & their resolution.
+                E.g. {"lrate": {"begin": 0.1, "end": 0.5, "bins": 5}}
+                Defaults to None.
+            integer (Optional[dict], optional):
+                Dictionary of integer-valued search variables & their resolution.
+                E.g. {"batch_size": {"begin": 1, "end": 5, "bins": 5}}
+                Defaults to None.
+            categorical (Optional[dict], optional):
+                Dictionary of categorical-valued search variables.
+                E.g. {"arch": ["mlp", "cnn"]}
+                Defaults to None.
+            search_config (dict, optional): Grid search hyperparameters.
+                Defaults to None.
+            maximize_objective (bool, optional): Whether to maximize objective.
+                Defaults to False.
+            fixed_params (Optional[dict], optional):
+                Fixed parameters that will be added to all configurations.
+                Defaults to None.
+
+    Returns:
+        Callable[Any]: _description_
     """
-    assert strategy_type in [
-        "Random",
-        "Grid",
-        "SMBO",
-        "Nevergrad",
-        "Coordinate",
-        "Halving",
-        "Hyperband",
-        "PBT",
-    ]
+    assert strategy_type in Strategies.keys()
 
-    if strategy_type == "Random":
-        strategy = RandomSearch(
-            real, integer, categorical, search_config, maximize_objective, fixed_params
-        )
-    elif strategy_type == "Grid":
-        strategy = GridSearch(real, integer, categorical, fixed_params)
-    elif strategy_type == "SMBO":
-        strategy = SMBOSearch(
-            real, integer, categorical, search_config, maximize_objective, fixed_params
-        )
-    elif strategy_type == "Nevergrad":
-        strategy = NevergradSearch(
-            real, integer, categorical, search_config, maximize_objective, fixed_params
-        )
-    elif strategy_type == "Coordinate":
-        strategy = CoordinateSearch(
-            real, integer, categorical, search_config, maximize_objective, fixed_params
-        )
-    elif strategy_type == "Halving":
-        strategy = HalvingSearch(
-            real, integer, categorical, search_config, maximize_objective, fixed_params
-        )
-    elif strategy_type == "Hyperband":
-        strategy = HyperbandSearch(
-            real, integer, categorical, search_config, maximize_objective, fixed_params
-        )
-    elif strategy_type == "PBT":
-        strategy = PBTSearch(
-            real, integer, categorical, search_config, maximize_objective, fixed_params
-        )
+    strategy = Strategies[strategy_type](
+        real,
+        integer,
+        categorical,
+        search_config,
+        maximize_objective,
+        fixed_params,
+    )
 
     def decorator(function):
         @functools.wraps(function)
         def wrapper(*args, **kwargs):
-            for iter_id in range(num_search_iters):
+            for _ in range(num_search_iters):
                 config = strategy.ask()
                 result = function(config, *args, **kwargs)
                 strategy.tell(config, [result])