Update the hyperparameter optimization example script (vwxyzjn#268)

* Hyperparameter optimization

* add gitignore

* pre-commit

* quick refactor

* Add docs

* pre-commit

* clarify docs

* update docs

* push changes

* push changes

* typo

* update dependencies

* Quick fix

* clarification

* Update docs on python version

* add test cases

* add tests

* update config

* update test cases

* Refactor docs

* Patch hyper parameter optimization example

Author: vwxyzjn

tuner_example.py

@@ -7,25 +7,25 @@
     metric="charts/episodic_return",
     metric_last_n_average_window=50,
     direction="maximize",
+    aggregation_type="average",
     target_scores={
         "CartPole-v1": [0, 500],
         "Acrobot-v1": [-500, 0],
     },
     params_fn=lambda trial: {
         "learning-rate": trial.suggest_loguniform("learning-rate", 0.0003, 0.003),
         "num-minibatches": trial.suggest_categorical("num-minibatches", [1, 2, 4]),
-        "update-epochs": trial.suggest_categorical("update-epochs", [1, 2, 4]),
+        "update-epochs": trial.suggest_categorical("update-epochs", [1, 2, 4, 8]),
         "num-steps": trial.suggest_categorical("num-steps", [5, 16, 32, 64, 128]),
         "vf-coef": trial.suggest_uniform("vf-coef", 0, 5),
         "max-grad-norm": trial.suggest_uniform("max-grad-norm", 0, 5),
-        "total-timesteps": 10000,
+        "total-timesteps": 100000,
         "num-envs": 16,
     },
     pruner=optuna.pruners.MedianPruner(n_startup_trials=5),
     sampler=optuna.samplers.TPESampler(),
-    # wandb_kwargs={"project": "cleanrl"},
 )
 tuner.tune(
-    num_trials=10,
+    num_trials=100,
     num_seeds=3,
 )