Update everest

src/everest/api/everest_data_api.py

@@ -1,9 +1,9 @@
 from collections import OrderedDict
 
 import pandas as pd
+from ert.storage import open_storage
 from seba_sqlite.snapshot import SebaSnapshot
 
-from ert.storage import open_storage
 from everest.config import EverestConfig
 from everest.detached import ServerStatus, everserver_status
 
@@ -158,9 +158,10 @@ def summary_values(self, batches=None, keys=None):
         data_frames = []
         storage = open_storage(self._config.storage_dir, "r")
         for batch_id in batches:
-            summary = storage.get_ensemble_by_name(
-                f"batch_{batch_id}"
-            ).load_all_summary_data()
+            case_name = f"batch_{batch_id}"
+            experiment = storage.get_experiment_by_name(f"experiment_{case_name}")
+            ensemble = experiment.get_ensemble_by_name(case_name)
+            summary = ensemble.load_all_summary_data()
             if not summary.empty:
                 columns = set(summary.columns)
                 if keys is not None:

src/everest/bin/everest_script.py

@@ -9,6 +9,7 @@
 
 from ert.config import ErtConfig
 from ert.storage import open_storage
+
 from everest.config import EverestConfig
 from everest.detached import (
     ServerStatus,

src/everest/bin/everload_script.py

@@ -10,6 +10,7 @@
 from ert import LibresFacade
 from ert.config import ErtConfig
 from ert.storage import open_storage
+
 from everest import MetaDataColumnNames as MDCN
 from everest import export
 from everest.config import EverestConfig
@@ -59,10 +60,11 @@ def everload_entry(args=None):
     backup_path = None
     if not os.path.isdir(storage_path):
         storage_path = None
-    elif not options.overwrite:
-        backup_path = storage_path + datetime.datetime.utcnow().strftime(
-            "__%Y-%m-%d_%H.%M.%S.%f"
-        )
+    else:  # storage_path exists, we may want to back it up
+        if not options.overwrite:
+            backup_path = storage_path + datetime.datetime.utcnow().strftime(
+                "__%Y-%m-%d_%H.%M.%S.%f"
+            )
 
     if not options.silent and not user_confirms(sim_dir, storage_path, backup_path):
         return
@@ -178,17 +180,18 @@ def reload_data(ever_config: EverestConfig, backup_path=None):
 
 def _internalize_batch(ert_config, batch_id, batch_data):
     facade = LibresFacade(ert_config)
-    batch_name = "batch_{}".format(batch_id)
+    case_name = "batch_{}".format(batch_id)
     batch_size = batch_data.shape[0]
     with open_storage(facade.enspath, "w") as storage:
-        file_system = storage.get_ensemble_by_name(batch_name)
+        experiment = storage.get_experiment_by_name(f"experiment_{case_name}")
+        ensemble = experiment.get_ensemble_by_name(case_name)
         # Everest artificially inflates the ensemble size as it is not possible to
         # add after the fact, therefore a batch is much smaller than the overall
         # ensemble size
         realizations = [True] * batch_size + [False] * (
             facade.get_ensemble_size() - batch_size
         )
-        facade.load_from_forward_model(file_system, realizations, 0)
+        facade.load_from_forward_model(ensemble, realizations, 0)
 
 
 if __name__ == "__main__":

src/everest/bin/main.py

@@ -3,6 +3,8 @@
 import logging
 import sys
 
+from ieverest.bin.ieverest_script import ieverest_entry
+
 from everest import __version__ as everest_version
 from everest import docs
 from everest.bin.config_branch_script import config_branch_entry
@@ -15,7 +17,6 @@
 from everest.bin.monitor_script import monitor_entry
 from everest.bin.visualization_script import visualization_entry
 from everest.util import configure_logger
-from ieverest.bin.ieverest_script import ieverest_entry
 
 
 def _create_dump_action(dumps, extended=False):

src/everest/bin/utils.py

@@ -8,8 +8,8 @@
 
 import colorama
 from colorama import Fore
-
 from ert.simulator.batch_simulator_context import Status
+
 from everest.config import EverestConfig
 from everest.detached import (
     OPT_PROGRESS_ID,

src/everest/config/everest_config.py

@@ -7,6 +7,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, List, Literal, Optional, Protocol, no_type_check
 
+from ert.config import ErtConfig
 from pydantic import (
     AfterValidator,
     BaseModel,
@@ -19,7 +20,6 @@
 from ruamel.yaml import YAML, YAMLError
 from typing_extensions import Annotated
 
-from ert.config import ErtConfig
 from everest.config.control_variable_config import ControlVariableGuessListConfig
 from everest.config.install_template_config import InstallTemplateConfig
 from everest.config.server_config import ServerConfig

src/everest/config/input_constraint_config.py

@@ -9,9 +9,9 @@ class InputConstraintConfig(BaseModel, extra="forbid"):  # type: ignore
 If we are trying to constrain only one control (i.e the z control) value:
 | input_constraints:
 | - weights:
-|   point_3D_0.x-0: 0
-|   point_3D_0.y-1: 0
-|   point_3D_0.z-2: 1
+|   point_3D.x-0: 0
+|   point_3D.y-1: 0
+|   point_3D.z-2: 1
 | upper_bound: 0.2
 
 Only control values (x, y, z) that satisfy the following equation will be allowed:
@@ -23,9 +23,9 @@ class InputConstraintConfig(BaseModel, extra="forbid"):  # type: ignore
         description="""**Example**
 | input_constraints:
 | - weights:
-|   point_3D_0.x-0: 1
-|   point_3D_0.y-1: 2
-|   point_3D_0.z-2: 3
+|   point_3D.x-0: 1
+|   point_3D.y-1: 2
+|   point_3D.z-2: 3
 | target: 4
 
 Only control values (x, y, z) that satisfy the following equation will be allowed:
@@ -37,9 +37,9 @@ class InputConstraintConfig(BaseModel, extra="forbid"):  # type: ignore
         description="""**Example**
 | input_constraints:
 | - weights:
-|   point_3D_0.x-0: 1
-|   point_3D_0.y-1: 2
-|   point_3D_0.z-2: 3
+|   point_3D.x-0: 1
+|   point_3D.y-1: 2
+|   point_3D.z-2: 3
 | lower_bound: 4
 
 Only control values (x, y, z) that satisfy the following
@@ -52,9 +52,9 @@ class InputConstraintConfig(BaseModel, extra="forbid"):  # type: ignore
         description="""**Example**
 | input_constraints:
 | - weights:
-|   point_3D_0.x-0: 1
-|   point_3D_0.y-1: 2
-|   point_3D_0.z-2: 3
+|   point_3D.x-0: 1
+|   point_3D.y-1: 2
+|   point_3D.z-2: 3
 | upper_bound: 4
 
 Only control values (x, y, z) that satisfy the following equation will be allowed:

src/everest/detached/__init__.py

@@ -10,11 +10,11 @@
 
 import pkg_resources
 import requests
+from ert import BatchContext, BatchSimulator
+from ert.config import ErtConfig, QueueSystem
 from seba_sqlite.exceptions import ObjectNotFoundError
 from seba_sqlite.snapshot import SebaSnapshot
 
-from ert import BatchContext, BatchSimulator
-from ert.config import ErtConfig, QueueSystem
 from everest.config import EverestConfig
 from everest.config_keys import ConfigKeys as CK
 from everest.simulator import JOB_FAILURE, JOB_SUCCESS, Status

src/everest/docs/__main__.py

@@ -5,7 +5,7 @@
 from everest.docs.generate_docs_from_config_spec import generate_docs_pydantic_to_rst
 
 committed_file = os.path.abspath(
-    relpath("..", "..", "docs", "source", "config_generated.rst")
+    relpath("..", "docs", "source", "config_generated.rst")
 )
 
 print(f"Writing new docs contents to {committed_file}")

src/everest/export.py

@@ -4,10 +4,10 @@
 from typing import Any, Dict, List, Optional, Set
 
 import pandas as pd
+from ert.storage import open_storage
 from pandas import DataFrame
 from seba_sqlite.snapshot import SebaSnapshot
 
-from ert.storage import open_storage
 from everest.config import EverestConfig
 from everest.strings import STORAGE_DIR
 
@@ -161,7 +161,9 @@ def get_internalized_keys(config: EverestConfig, batch_ids: Optional[Set[int]] =
     internal_keys: Set = set()
     with open_storage(config.storage_dir, "r") as storage:
         for batch_id in batch_ids:
-            ensemble = storage.get_ensemble_by_name("batch_{}".format(batch_id))
+            case_name = f"batch_{batch_id}"
+            experiment = storage.get_experiment_by_name(f"experiment_{case_name}")
+            ensemble = experiment.get_ensemble_by_name(case_name)
             if not internal_keys:
                 internal_keys = set(ensemble.get_summary_keyset())
             else:
@@ -310,9 +312,10 @@ def _load_simulation_data(
     with open_storage(ens_path, "r") as storage:
         # pylint: disable=unnecessary-lambda-assignment
         def load_batch_by_id():
-            return storage.get_ensemble_by_name(
-                f"batch_{batch}"
-            ).load_all_summary_data()
+            case_name = f"batch_{batch}"
+            experiment = storage.get_experiment_by_name(f"experiment_{case_name}")
+            ensemble = experiment.get_ensemble_by_name(case_name)
+            return ensemble.load_all_summary_data()
 
         batches = {elem[MetaDataColumnNames.BATCH] for elem in metadata}
         batch_data = []

src/everest/jobs/scripts/recovery_factor

@@ -20,8 +20,8 @@ def _build_argument_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--output_file",
         required=False,
-        default="rf_0",
-        help="the output file, defaults to rf_0",
+        default="rf",
+        help="the output file, defaults to rf",
     )
     return parser
 

src/everest/queue_driver/queue_driver.py

@@ -1,6 +1,7 @@
 from typing import Any, List, Optional, Tuple
 
 from ert.config import QueueSystem
+
 from everest.config import EverestConfig
 from everest.config.simulator_config import SimulatorConfig
 from everest.config_keys import ConfigKeys

src/everest/simulator/__init__.py

@@ -1,4 +1,5 @@
 from ert.simulator.batch_simulator_context import Status
+
 from everest.simulator.simulator import Simulator
 
 JOB_SUCCESS = "Success"

src/everest/simulator/simulator.py

@@ -4,13 +4,13 @@
 from typing import Any, DefaultDict, Dict, List, Mapping, Optional, Tuple, Union
 
 import numpy as np
+from ert import BatchSimulator, WorkflowRunner
+from ert.config import ErtConfig, HookRuntime
+from ert.storage import open_storage
 from numpy import float64
 from numpy._typing import NDArray
 from ropt.evaluator import EvaluatorContext, EvaluatorResult
 
-from ert import BatchSimulator, WorkflowRunner
-from ert.config import ErtConfig, HookRuntime
-from ert.storage import open_storage
 from everest.config import EverestConfig
 from everest.config.control_variable_config import (
     ControlVariableConfig,