Allow use of Azure VM Service for only remote exec.

mlos_bench/mlos_bench/config/schemas/services/remote/azure/azure-service-subschema.json → mlos_bench/mlos_bench/config/schemas/services/remote/azure/azure-deployment-service-subschema.json

@@ -1,10 +1,10 @@
 {
     "$schema": "https://json-schema.org/draft/2020-12/schema",
-    "$id": "https://raw.githubusercontent.com/microsoft/MLOS/main/mlos_bench/mlos_bench/config/schemas/services/remote/azure/azure-service-subschema.json",
-    "title": "mlos_bench Azure Base Service config",
-    "description": "Base config schema for an mlos_bench Azure Service",
+    "$id": "https://raw.githubusercontent.com/microsoft/MLOS/main/mlos_bench/mlos_bench/config/schemas/services/remote/azure/azure-deployment-service-subschema.json",
+    "title": "mlos_bench Azure Base Deployment Service config",
+    "description": "Base config schema for an mlos_bench Azure Deployment Services",
     "$defs": {
-        "azure_service_config": {
+        "azure_deployment_service_config": {
             "type": "object",
             "allOf": [
                 {
@@ -21,8 +21,8 @@
                             "type": "string"
                         },
                         "deploymentTemplatePath": {
-                            "description": "Path to an ARM template file.",
-                            "type": "string",
+                            "description": "Path to an ARM template file, or null if it should be skipped.",
+                            "type": ["string", "null"],
                             "pattern": "[.]json[c]?$"
                         },
                         "deploymentTemplateParameters": {
@@ -49,10 +49,7 @@
                             },
                             "minProperties": 1
                         }
-                    },
-                    "required": [
-                        "deploymentTemplatePath"
-                    ]
+                    }
                 }
             ]
         }

mlos_bench/mlos_bench/config/schemas/services/remote/azure/azure-network-service-subschema.json

@@ -15,7 +15,11 @@
             "type": "object",
             "allOf": [
                 {
-                    "$ref": "./azure-service-subschema.json#/$defs/azure_service_config"
+                    "$ref": "./azure-deployment-service-subschema.json#/$defs/azure_deployment_service_config"
+                },
+                {
+                    "$comment": "Unlike VM service, which can still do remote exec, network service needs a deployment template.",
+                    "required": ["deploymentTemplatePath"]
                 }
             ],
             "unevaluatedProperties": false

mlos_bench/mlos_bench/config/schemas/services/remote/azure/azure-vm-service-subschema.json

@@ -15,7 +15,7 @@
             "type": "object",
             "allOf": [
                 {
-                    "$ref": "./azure-service-subschema.json#/$defs/azure_service_config"
+                    "$ref": "./azure-deployment-service-subschema.json#/$defs/azure_deployment_service_config"
                 },
                 {
                     "properties": {
@@ -34,5 +34,5 @@
             "unevaluatedProperties": false
         }
     },
-    "required": ["class", "config"]
+    "required": ["class"]
 }

mlos_bench/mlos_bench/services/remote/azure/azure_services.py → mlos_bench/mlos_bench/services/remote/azure/azure_deployment_services.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License.
 #
 """
-Base class for certain Azure Services classes.
+Base class for certain Azure Services classes that do deployments.
 """
 
 import abc
@@ -25,9 +25,9 @@
 _LOG = logging.getLogger(__name__)
 
 
-class AzureService(Service, metaclass=abc.ABCMeta):
+class AzureDeploymentService(Service, metaclass=abc.ABCMeta):
     """
-    Helper methods to manage Azure resources via REST APIs.
+    Helper methods to manage and deploy Azure resources via REST APIs.
     """
 
     _POLL_INTERVAL = 4     # seconds
@@ -73,8 +73,6 @@ def __init__(self,
         check_required_params(self.config, [
             "subscription",
             "resourceGroup",
-            "deploymentTemplatePath",
-            "deploymentTemplateParameters",
         ])
 
         # These parameters can come from command line as strings, so conversion is needed.
@@ -84,15 +82,20 @@ def __init__(self,
         self._total_retries = int(self.config.get("requestTotalRetries", self._REQUEST_TOTAL_RETRIES))
         self._backoff_factor = float(self.config.get("requestBackoffFactor", self._REQUEST_RETRY_BACKOFF_FACTOR))
 
-        # TODO: Provide external schema validation?
-        template = self.config_loader_service.load_config(
-            self.config['deploymentTemplatePath'], schema_type=None)
-        assert template is not None and isinstance(template, dict)
-        self._deploy_template = template
-
-        # Allow for recursive variable expansion as we do with global params and const_args.
-        deploy_params = DictTemplater(self.config['deploymentTemplateParameters']).expand_vars(extra_source_dict=global_config)
-        self._deploy_params = merge_parameters(dest=deploy_params, source=global_config)
+        self._deploy_template = {}
+        self._deploy_params = {}
+        if self.config.get("deploymentTemplatePath") is not None:
+            # TODO: Provide external schema validation?
+            template = self.config_loader_service.load_config(
+                self.config['deploymentTemplatePath'], schema_type=None)
+            assert template is not None and isinstance(template, dict)
+            self._deploy_template = template
+
+            # Allow for recursive variable expansion as we do with global params and const_args.
+            deploy_params = DictTemplater(self.config['deploymentTemplateParameters']).expand_vars(extra_source_dict=global_config)
+            self._deploy_params = merge_parameters(dest=deploy_params, source=global_config)
+        else:
+            _LOG.info("No deploymentTemplatePath provided. Deployment services will be unavailable.")
 
     @property
     def deploy_params(self) -> dict:
@@ -407,6 +410,8 @@ def _provision_resource(self, params: dict) -> Tuple[Status, dict]:
             parameters extracted from the response JSON, or {} if the status is FAILED.
             Status is one of {PENDING, SUCCEEDED, FAILED}
         """
+        if not self._deploy_template:
+            raise ValueError(f"Missing deployment template: {self}")
         params = self._set_default_params(params)
         config = merge_parameters(dest=self.config.copy(), source=params, required_keys=["deploymentName"])
         _LOG.info("Deploy: %s :: %s", config["deploymentName"], params)

mlos_bench/mlos_bench/services/remote/azure/azure_network_services.py

@@ -12,14 +12,14 @@
 
 from mlos_bench.environments.status import Status
 from mlos_bench.services.base_service import Service
-from mlos_bench.services.remote.azure.azure_services import AzureService
+from mlos_bench.services.remote.azure.azure_deployment_services import AzureDeploymentService
 from mlos_bench.services.types.network_provisioner_type import SupportsNetworkProvisioning
 from mlos_bench.util import merge_parameters
 
 _LOG = logging.getLogger(__name__)
 
 
-class AzureNetworkService(AzureService, SupportsNetworkProvisioning):
+class AzureNetworkService(AzureDeploymentService, SupportsNetworkProvisioning):
     """
     Helper methods to manage Virtual Networks on Azure.
     """
@@ -67,6 +67,9 @@ def __init__(self,
                 self.wait_network_deployment,
             ])
         )
+        if not self._deploy_template:
+            raise ValueError("AzureNetworkService requires a deployment template:\n"
+                             + f"config={config}\nglobal_config={global_config}")
 
     def _set_default_params(self, params: dict) -> dict:    # pylint: disable=no-self-use
         # Try and provide a semi sane default for the deploymentName if not provided

mlos_bench/mlos_bench/services/remote/azure/azure_vm_services.py

@@ -15,7 +15,7 @@
 
 from mlos_bench.environments.status import Status
 from mlos_bench.services.base_service import Service
-from mlos_bench.services.remote.azure.azure_services import AzureService
+from mlos_bench.services.remote.azure.azure_deployment_services import AzureDeploymentService
 from mlos_bench.services.types.remote_exec_type import SupportsRemoteExec
 from mlos_bench.services.types.host_provisioner_type import SupportsHostProvisioning
 from mlos_bench.services.types.host_ops_type import SupportsHostOps
@@ -25,7 +25,7 @@
 _LOG = logging.getLogger(__name__)
 
 
-class AzureVMService(AzureService, SupportsHostProvisioning, SupportsHostOps, SupportsOSOps, SupportsRemoteExec):
+class AzureVMService(AzureDeploymentService, SupportsHostProvisioning, SupportsHostOps, SupportsOSOps, SupportsRemoteExec):
     """
     Helper methods to manage VMs on Azure.
     """

mlos_bench/mlos_bench/tests/config/schemas/services/test-cases/good/partial/azure_vm_service-partial-no-deployment.jsonc

@@ -0,0 +1,5 @@
+{
+    "class": "mlos_bench.services.remote.azure.AzureVMService"
+    // This version can only support remote exec when subscription, resourceGroup,
+    // and vmName are in the parameters for ther environment.
+}

mlos_bench/mlos_bench/tests/config/schemas/services/test_services_schemas.py

@@ -17,7 +17,7 @@
 from mlos_bench.services.base_service import Service
 from mlos_bench.services.config_persistence import ConfigPersistenceService
 from mlos_bench.services.local.temp_dir_context import TempDirContextService
-from mlos_bench.services.remote.azure.azure_services import AzureService
+from mlos_bench.services.remote.azure.azure_deployment_services import AzureDeploymentService
 from mlos_bench.services.remote.ssh.ssh_service import SshService
 
 from mlos_bench.tests import try_resolve_class_name
@@ -39,7 +39,7 @@
 NON_CONFIG_SERVICE_CLASSES = {
     ConfigPersistenceService,   # configured thru the launcher cli args
     TempDirContextService,      # ABCMeta abstract class, but no good way to test that dynamically in Python.
-    AzureService,               # ABCMeta abstract base class
+    AzureDeploymentService,     # ABCMeta abstract base class
     SshService,                 # ABCMeta abstract base class
 }
 

mlos_bench/mlos_bench/tests/services/remote/azure/azure_network_services_test.py

@@ -13,6 +13,7 @@
 
 from mlos_bench.environments.status import Status
 
+from mlos_bench.services.remote.azure.azure_auth import AzureAuthService
 from mlos_bench.services.remote.azure.azure_network_services import AzureNetworkService
 
 from mlos_bench.tests.services.remote.azure import make_httplib_json_response
@@ -66,7 +67,7 @@ def test_wait_network_deployment_retry(mock_getconn: MagicMock,
         (401, Status.SUCCEEDED),
         (404, Status.SUCCEEDED),
     ])
-@patch("mlos_bench.services.remote.azure.azure_services.requests")
+@patch("mlos_bench.services.remote.azure.azure_deployment_services.requests")
 # pylint: disable=too-many-arguments
 def test_network_operation_status(mock_requests: MagicMock,
                                   azure_network_service: AzureNetworkService,
@@ -87,3 +88,25 @@ def test_network_operation_status(mock_requests: MagicMock,
         (status, _) = operation({}) if accepts_params else operation()
     (status, _) = operation({"vnetName": "test-vnet"}) if accepts_params else operation()
     assert status == operation_status
+
+
+@pytest.fixture
+def test_azure_network_service_no_deployment_template(azure_auth_service: AzureAuthService) -> None:
+    """
+    Tests creating a network services without a deployment template (should fail).
+    """
+    with pytest.raises(ValueError):
+        _ = AzureNetworkService(config={
+            "deploymentTemplatePath": None,
+            "deploymentTemplateParameters": {
+                "location": "westus2",
+            },
+        }, parent=azure_auth_service)
+
+    with pytest.raises(ValueError):
+        _ = AzureNetworkService(config={
+            # "deploymentTemplatePath": None,
+            "deploymentTemplateParameters": {
+                "location": "westus2",
+            },
+        }, parent=azure_auth_service)

mlos_bench/mlos_bench/tests/services/remote/azure/azure_vm_services_test.py

@@ -123,7 +123,7 @@ def test_azure_vm_service_custom_data(azure_auth_service: AzureAuthService) -> N
         (401, Status.FAILED),
         (404, Status.FAILED),
     ])
-@patch("mlos_bench.services.remote.azure.azure_services.requests")
+@patch("mlos_bench.services.remote.azure.azure_deployment_services.requests")
 # pylint: disable=too-many-arguments
 def test_vm_operation_status(mock_requests: MagicMock,
                              azure_vm_service: AzureVMService,
@@ -146,8 +146,23 @@ def test_vm_operation_status(mock_requests: MagicMock,
     assert status == operation_status
 
 
-@patch("mlos_bench.services.remote.azure.azure_services.time.sleep")
-@patch("mlos_bench.services.remote.azure.azure_services.requests.Session")
+@pytest.mark.parametrize(
+    ("operation_name", "accepts_params"), [
+        ("provision_host", True),
+    ])
+def test_vm_operation_invalid(azure_vm_service_remote_exec_only: AzureVMService,
+                              operation_name: str,
+                              accepts_params: bool) -> None:
+    """
+    Test VM operation status for an incomplete service config.
+    """
+    operation = getattr(azure_vm_service_remote_exec_only, operation_name)
+    with pytest.raises(ValueError):
+        (_, _) = operation({"vmName": "test-vm"}) if accepts_params else operation()
+
+
+@patch("mlos_bench.services.remote.azure.azure_deployment_services.time.sleep")
+@patch("mlos_bench.services.remote.azure.azure_deployment_services.requests.Session")
 def test_wait_vm_operation_ready(mock_session: MagicMock, mock_sleep: MagicMock,
                                  azure_vm_service: AzureVMService) -> None:
     """
@@ -175,7 +190,7 @@ def test_wait_vm_operation_ready(mock_session: MagicMock, mock_sleep: MagicMock,
     assert status.is_succeeded()
 
 
-@patch("mlos_bench.services.remote.azure.azure_services.requests.Session")
+@patch("mlos_bench.services.remote.azure.azure_deployment_services.requests.Session")
 def test_wait_vm_operation_timeout(mock_session: MagicMock,
                                    azure_vm_service: AzureVMService) -> None:
     """
@@ -240,7 +255,7 @@ def test_wait_vm_operation_retry(mock_getconn: MagicMock,
         (404, Status.FAILED),
     ])
 @patch("mlos_bench.services.remote.azure.azure_vm_services.requests")
-def test_remote_exec_status(mock_requests: MagicMock, azure_vm_service: AzureVMService,
+def test_remote_exec_status(mock_requests: MagicMock, azure_vm_service_remote_exec_only: AzureVMService,
                             http_status_code: int, operation_status: Status) -> None:
     """
     Test waiting for completion of the remote execution on Azure.
@@ -254,14 +269,14 @@ def test_remote_exec_status(mock_requests: MagicMock, azure_vm_service: AzureVMS
     })
     mock_requests.post.return_value = mock_response
 
-    status, _ = azure_vm_service.remote_exec(script, config={"vmName": "test-vm"}, env_params={})
+    status, _ = azure_vm_service_remote_exec_only.remote_exec(script, config={"vmName": "test-vm"}, env_params={})
 
     assert status == operation_status
 
 
 @patch("mlos_bench.services.remote.azure.azure_vm_services.requests")
 def test_remote_exec_headers_output(mock_requests: MagicMock,
-                                    azure_vm_service: AzureVMService) -> None:
+                                    azure_vm_service_remote_exec_only: AzureVMService) -> None:
     """
     Check if HTTP headers from the remote execution on Azure are correct.
     """
@@ -279,7 +294,7 @@ def test_remote_exec_headers_output(mock_requests: MagicMock,
     })
     mock_requests.post.return_value = mock_response
 
-    _, cmd_output = azure_vm_service.remote_exec(script, config={"vmName": "test-vm"}, env_params={
+    _, cmd_output = azure_vm_service_remote_exec_only.remote_exec(script, config={"vmName": "test-vm"}, env_params={
         "param_1": 123,
         "param_2": "abc",
     })
@@ -315,7 +330,7 @@ def test_remote_exec_headers_output(mock_requests: MagicMock,
         (Status.PENDING, {}, {}),
         (Status.FAILED, {}, {}),
     ])
-def test_get_remote_exec_results(azure_vm_service: AzureVMService, operation_status: Status,
+def test_get_remote_exec_results(azure_vm_service_remote_exec_only: AzureVMService, operation_status: Status,
                                  wait_output: dict, results_output: dict) -> None:
     """
     Test getting the results of the remote execution on Azure.
@@ -325,9 +340,9 @@ def test_get_remote_exec_results(azure_vm_service: AzureVMService, operation_sta
     mock_wait_host_operation = MagicMock()
     mock_wait_host_operation.return_value = (operation_status, wait_output)
     # azure_vm_service.wait_host_operation = mock_wait_host_operation
-    setattr(azure_vm_service, "wait_host_operation", mock_wait_host_operation)
+    setattr(azure_vm_service_remote_exec_only, "wait_host_operation", mock_wait_host_operation)
 
-    status, cmd_output = azure_vm_service.get_remote_exec_results(params)
+    status, cmd_output = azure_vm_service_remote_exec_only.get_remote_exec_results(params)
 
     assert status == operation_status
     assert mock_wait_host_operation.call_args[0][0] == params

mlos_bench/mlos_bench/tests/services/remote/azure/conftest.py

@@ -80,6 +80,21 @@ def azure_vm_service(azure_auth_service: AzureAuthService) -> AzureVMService:
     }, parent=azure_auth_service)
 
 
+@pytest.fixture
+def azure_vm_service_remote_exec_only(azure_auth_service: AzureAuthService) -> AzureVMService:
+    """
+    Creates a dummy Azure VM service with no deployment template.
+    """
+    return AzureVMService(config={
+        "subscription": "TEST_SUB",
+        "resourceGroup": "TEST_RG",
+        "pollInterval": 1,
+        "pollTimeout": 2,
+    }, global_config={
+        "vmName": "test-vm",  # Should come from the upper-level config
+    }, parent=azure_auth_service)
+
+
 @pytest.fixture
 def azure_fileshare(config_persistence_service: ConfigPersistenceService) -> AzureFileShareService:
     """