Merge pull request #32 from solace-iot-team/feature-au

Feature au

Author: ricardojosegomezulmke

.github/workflows/ansible-test.yml

@@ -31,7 +31,7 @@ jobs:
       fail-fast: false
       matrix:
         # 3.6 is always tested
-        python-version: [ '3.8' ]
+        python-version: [ '3.8.10' ]
         ansible-version: [
           "ansible>=2.10.3,<2.11", # this is 2.10
           "ansible>=4.10.0,<5.0.0", # this is 2.11

.github/workflows/integration.yml

@@ -151,6 +151,8 @@ jobs:
           export PROJECT_HOME="$GITHUB_WORKSPACE"
           export SOLACE_CLOUD_API_TOKEN_ALL_PERMISSIONS="${{ secrets.SOLACE_CLOUD_API_TOKEN_ALL_PERMISSIONS }}"
           export SOLACE_CLOUD_API_TOKEN_RESTRICTED_PERMISSIONS="${{ secrets.SOLACE_CLOUD_API_TOKEN_RESTRICTED_PERMISSIONS }}"
+          export SOLACE_CLOUD_API_TOKEN_US="${{ secrets.SOLACE_CLOUD_API_TOKEN_ALL_PERMISSIONS }}"
+          export SOLACE_CLOUD_API_TOKEN_AU="${{ secrets.SOLACE_CLOUD_API_TOKEN_ALL_PERMISSIONS_SOLACE_AU }}"
           export LOG_DIR="$GITHUB_WORKSPACE/${TEST_RUNNER_LOGS_DIR}"
           export AZURE_PROJECT_NAME="asct-wf"
           # source $PROJECT_HOME/test-runner/source.env.integration.core.sh

ReleaseNotes.md

@@ -1,5 +1,25 @@
 # Release Notes
 
+## Version 1.9.0
+
+**Framework:**
+  * **Support for Solace Cloud Home: US & AU**
+    - affects all modules which use the Solace Cloud Api
+    - new env var: ANSIBLE_SOLACE_SOLACE_CLOUD_HOME='AU' or 'US'
+    - new optional module parameter: solace_cloud_home: choices=['us', 'au']
+    - logic:
+      - default: 'US'
+      - if module parameter is set: use module parameter
+      - else check env var
+        - if set, all modules which use the Solace Cloud API will switch their base url according to the home cloud
+  * **Fixed timeout for Solace Cloud Api Post request polling**
+    - timeout parameter now passed through
+    - min polling time set to 5 mins
+  * **Fixed error on outstanding requests for serviceId for Solace Cloud Api**
+    - check if serviceId has any outstanding requests before submitting a new one
+    - uses same timeout as the actual request
+
+
 ## Version 1.8.0
 New modules to manage cert authorities & support for ansible_core 2.12.
 

devel/devel.requirements.txt

@@ -3,7 +3,7 @@ pycodestyle
 pyyaml
 voluptuous
 yamllint
-rstcheck
+rstcheck==3.3.1
 Sphinx
 sphinx-rtd-theme
 sphinxcontrib-contentui

src/ansible_collections/solace/pubsub_plus/docs/source/inventory-files.rst

@@ -201,7 +201,7 @@ Inventory File for a Solace Cloud Account
         ansible_connection: local
         broker_type: solace_cloud
         solace_cloud_api_token: the-token
-
+        solace_cloud_home: us
 
 When your Playbook only manages a Solace Cloud account, for example to start/stop services, all you need in the Inventory File is the api token with the correct permissions.
 

src/ansible_collections/solace/pubsub_plus/docs/source/solace_cloud_modules.rst

@@ -6,6 +6,30 @@ This can either be provided in the inventory file or as an environment variable
 
 See a detailed discussion here: :ref:`inventory_files_solace_cloud_account`.
 
+Setting Solace Cloud Home Cloud Region
+--------------------------------------
+
+Ansible Solace currently supports two Home Cloud Regions: `us` & `au`.
+
+You can either use an environment variable to set the home region for all solace cloud modules and/or you can specify an additional parameter module by module.
+
+Environment Variable: `ANSIBLE_SOLACE_SOLACE_CLOUD_HOME` , values: ["US", "AU"].
+
+Module parameter:
+
+* solace_cloud_home: choices=['us', 'au']
+
+The logic of choosing the home region is as follows:
+
+.. code-block:: none
+
+  - default: US
+  - if module parameter solace_cloud_home is set, use value from module parameter
+  - else
+    - check if env var is set and use value from env
+  - else
+    - use default
+
 
 Module Reference
 ----------------

src/ansible_collections/solace/pubsub_plus/galaxy.yml

@@ -8,7 +8,7 @@ namespace: solace
 name: pubsub_plus
 
 # The version of the collection. Must be compatible with semantic versioning
-version: 1.8.0
+version: 1.9.0
 
 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection
 readme: README.md

src/ansible_collections/solace/pubsub_plus/plugins/doc_fragments/solace.py

@@ -104,6 +104,16 @@ class ModuleDocFragment(object):
 
     BROKER_CONFIG_SOLACE_CLOUD = r'''
 options:
+  solace_cloud_home:
+    description: The Solace Cloud home region.
+    type: str
+    required: false
+    choices:
+      - us
+      - au
+      - US
+      - AU
+      - ''
   solace_cloud_api_token:
     description:
       - The API Token.
@@ -122,6 +132,16 @@ class ModuleDocFragment(object):
 
     BROKER_CONFIG_SOLACE_CLOUD_MANDATORY = r'''
 options:
+  solace_cloud_home:
+    description: The Solace Cloud home region.
+    type: str
+    required: false
+    choices:
+      - us
+      - au
+      - US
+      - AU
+      - ''
   solace_cloud_api_token:
     description:
       - The API Token.
@@ -140,6 +160,16 @@ class ModuleDocFragment(object):
 
     SOLACE_CLOUD_CONFIG_SOLACE_CLOUD = r'''
 options:
+  solace_cloud_home:
+    description: The Solace Cloud home region.
+    type: str
+    required: false
+    choices:
+      - us
+      - au
+      - US
+      - AU
+      - ''
   solace_cloud_api_token:
     description:
       - The API Token.
@@ -148,7 +178,7 @@ class ModuleDocFragment(object):
     required: true
     aliases: [api_token]
   timeout:
-    description: Connection timeout in seconds for the http request.
+    description: Connection timeout in seconds for the http request or overall call interaction timeout for Solace Cloud API.
     required: false
     default: 60
     type: int

src/ansible_collections/solace/pubsub_plus/plugins/module_utils/solace_api.py

@@ -2,12 +2,13 @@
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
 from __future__ import (absolute_import, division, print_function)
+import os
 __metaclass__ = type
 
 from ansible_collections.solace.pubsub_plus.plugins.module_utils import solace_sys
 from ansible_collections.solace.pubsub_plus.plugins.module_utils.solace_utils import SolaceUtils
 from ansible_collections.solace.pubsub_plus.plugins.module_utils.solace_consts import SolaceTaskOps
-from ansible_collections.solace.pubsub_plus.plugins.module_utils.solace_error import SolaceCloudApiResponseDataError, SolaceInternalErrorAbstractMethod, SolaceApiError, SolaceParamsValidationError
+from ansible_collections.solace.pubsub_plus.plugins.module_utils.solace_error import SolaceCloudApiResponseDataError, SolaceEnvVarError, SolaceInternalErrorAbstractMethod, SolaceApiError, SolaceParamsValidationError
 from ansible_collections.solace.pubsub_plus.plugins.module_utils.solace_error import SolaceInternalError
 from ansible_collections.solace.pubsub_plus.plugins.module_utils.solace_task_config import SolaceTaskConfig, SolaceTaskBrokerConfig, SolaceTaskSolaceCloudConfig
 from ansible.module_utils.basic import AnsibleModule
@@ -489,7 +490,12 @@ def get_objects(self, config: SolaceTaskBrokerConfig, xml_cmd: str, reponse_list
 
 class SolaceCloudApi(SolaceApi):
 
-    API_BASE_PATH = "https://api.solace.cloud/api/v0"
+    ENV_VAR_ANSIBLE_SOLACE_SOLACE_CLOUD_HOME = "ANSIBLE_SOLACE_SOLACE_CLOUD_HOME"
+    ANSIBLE_SOLACE_SOLACE_CLOUD_HOME_US = "us"
+    ANSIBLE_SOLACE_SOLACE_CLOUD_HOME_AU = "au"
+    API_BASE_PATH_US = "https://api.solace.cloud/api/v0"
+    API_BASE_PATH_AU = "https://api.solacecloud.com.au/api/v0"
+
     API_DATA_CENTERS = "datacenters"
     API_SERVICES = "services"
     API_REQUESTS = "requests"
@@ -498,6 +504,28 @@ def __init__(self, module: AnsibleModule):
         super().__init__(module)
         return
 
+    def get_api_base_path(self, config: SolaceTaskSolaceCloudConfig) -> str:
+        solace_cloud_home_value = self.ANSIBLE_SOLACE_SOLACE_CLOUD_HOME_US
+
+        if config.solace_cloud_home is not None and config.solace_cloud_home != '':
+            solace_cloud_home_value = config.solace_cloud_home.lower()
+        else:
+            solaceCloudHomeEnvVal = os.getenv(
+                self.ENV_VAR_ANSIBLE_SOLACE_SOLACE_CLOUD_HOME)
+            if solaceCloudHomeEnvVal is not None and solaceCloudHomeEnvVal != '':
+                if solaceCloudHomeEnvVal.lower() == self.ANSIBLE_SOLACE_SOLACE_CLOUD_HOME_US:
+                    solace_cloud_home_value = self.ANSIBLE_SOLACE_SOLACE_CLOUD_HOME_US
+                elif solaceCloudHomeEnvVal.lower() == self.ANSIBLE_SOLACE_SOLACE_CLOUD_HOME_AU:
+                    solace_cloud_home_value = self.ANSIBLE_SOLACE_SOLACE_CLOUD_HOME_AU
+                else:
+                    raise SolaceEnvVarError(self.ENV_VAR_ANSIBLE_SOLACE_SOLACE_CLOUD_HOME,
+                                            solaceCloudHomeEnvVal,
+                                            f"allowed values: {self.ANSIBLE_SOLACE_SOLACE_CLOUD_HOME_US}, {self.ANSIBLE_SOLACE_SOLACE_CLOUD_HOME_AU}")
+        if solace_cloud_home_value == self.ANSIBLE_SOLACE_SOLACE_CLOUD_HOME_US:
+            return self.API_BASE_PATH_US
+        else:
+            return self.API_BASE_PATH_AU
+
     def get_auth(self, config: SolaceTaskBrokerConfig) -> str:
         return config.get_solace_cloud_auth()
 
@@ -524,7 +552,7 @@ def handle_bad_response(self, resp, module_op):
     def get_data_centers(self, config: SolaceTaskSolaceCloudConfig) -> list:
         # GET /api/v0/datacenters
         resp = self.make_get_request(
-            config, [self.API_BASE_PATH, self.API_DATA_CENTERS], query_params=None)
+            config, [self.get_api_base_path(config), self.API_DATA_CENTERS], query_params=None)
         return resp
 
     def _transform_service(self, service: dict) -> dict:
@@ -546,7 +574,7 @@ def get_services(self, config: SolaceTaskSolaceCloudConfig) -> list:
         module_op = SolaceTaskOps.OP_READ_OBJECT_LIST
         try:
             _resp = self.make_get_request(
-                config, [self.API_BASE_PATH, self.API_SERVICES], module_op)
+                config, [self.get_api_base_path(config), self.API_SERVICES], module_op)
         except SolaceApiError as e:
             resp = e.get_resp()
             # TODO: what is the code if solace cloud account has 0 services?
@@ -582,7 +610,7 @@ def get_service(self, config: SolaceTaskSolaceCloudConfig, service_id: str) -> d
         module_op = SolaceTaskOps.OP_READ_OBJECT
         try:
             _resp = self.make_get_request(
-                config, [self.API_BASE_PATH, self.API_SERVICES, service_id], module_op)
+                config, [self.get_api_base_path(config), self.API_SERVICES, service_id], module_op)
         except SolaceApiError as e:
             resp = e.get_resp()
             if resp['status_code'] == 404:
@@ -603,7 +631,7 @@ def create_service(self, config: SolaceTaskSolaceCloudConfig, wait_timeout_minut
         # POST https://api.solace.cloud/api/v0/services
         module_op = SolaceTaskOps.OP_CREATE_OBJECT
         resp = self.make_post_request(
-            config, [self.API_BASE_PATH, self.API_SERVICES], data, module_op)
+            config, [self.get_api_base_path(config), self.API_SERVICES], data, module_op)
         _service_id = resp['serviceId']
         if wait_timeout_minutes > 0:
             res = self.wait_for_service_create_completion(
@@ -671,7 +699,7 @@ def wait_for_service_create_completion(self, config: SolaceTaskSolaceCloudConfig
 
     def delete_service(self, config: SolaceTaskSolaceCloudConfig, service_id: str) -> dict:
         # DELETE https://api.solace.cloud/api/v0/services/{{serviceId}}
-        path_array = [SolaceCloudApi.API_BASE_PATH,
+        path_array = [self.get_api_base_path(config),
                       SolaceCloudApi.API_SERVICES, service_id]
         return self.make_delete_request(config, path_array)
 
@@ -702,7 +730,7 @@ def get_object_settings(self, config: SolaceTaskBrokerConfig, path_array: list)
     def get_service_request_status(self, config: SolaceTaskBrokerConfig, service_id: str, request_id: str):
         module_op = SolaceTaskOps.OP_READ_OBJECT
         # GET https://api.solace.cloud/api/v0/services/{paste-your-serviceId-here}/requests/{{requestId}}
-        path_array = [self.API_BASE_PATH, self.API_SERVICES,
+        path_array = [self.get_api_base_path(config), self.API_SERVICES,
                       service_id, self.API_REQUESTS, request_id]
         resp = self.make_get_request(config, path_array, module_op)
         # resp may not yet contain 'adminProgress' depending on whether this creation has started yet
@@ -711,16 +739,62 @@ def get_service_request_status(self, config: SolaceTaskBrokerConfig, service_id:
             resp['adminProgress'] = 'inProgress'
         return resp
 
+    def wait_for_service_requests_to_finish(self, config: SolaceTaskBrokerConfig, timeout_minutes: int, service_id: str):
+        module_op = SolaceTaskOps.OP_READ_OBJECT
+        # GET https://api.solace.cloud/api/v0/services/{paste-your-serviceId-here}/requests
+        # returns list of dicts,
+        # - check all elements,
+        # - if "adminProgress" == "inProgress", wait and try again
+        # - raise SolaceApiError if timeout
+        are_all_completed = False
+        try_count = -1
+        delay = 30  # seconds
+        max_retries = (timeout_minutes * 60) // delay
+        while not are_all_completed and try_count < max_retries:
+            path_array = [self.get_api_base_path(config), self.API_SERVICES,
+                          service_id, self.API_REQUESTS]
+            resp = self.make_get_request(config, path_array, module_op)
+            # logging.debug(f"wait_for_service_requests_to_finish(): resp=\n{json.dumps(resp, indent=2)}")
+            # iterate through list to check if any adminProgress == inProgress
+            # use generator:
+            matches = (
+                respElem for respElem in resp if respElem['adminProgress'] == 'inProgress')
+            notCompletedElement = next(matches, None)
+            if notCompletedElement is None:
+                are_all_completed = True
+            try_count += 1
+            if not are_all_completed and timeout_minutes > 0:
+                time.sleep(delay)
+
+        if not are_all_completed:
+            msg = [
+                "timeout waiting for all outstanding service requests to be completed",
+                f"timeout(mins)={timeout_minutes}",
+                "request in progress:",
+                str(notCompletedElement)]
+            raise SolaceApiError(
+                resp, msg, self.get_module()._name, module_op)
+        return
+
     def make_service_post_request(self, config: SolaceTaskBrokerConfig, path_array: list, service_id: str, json_body, module_op):
+
+        timeout_minutes = config.get_timeout() // 60
+        # set min timeout to 5 mins
+        timeout_minutes = max(timeout_minutes, 5)
+
+        # check if there are any jobs still running against this service and wait until completed
+        self.wait_for_service_requests_to_finish(
+            config, timeout_minutes, service_id)
+
+        # now make the request
         resp = self.make_request(config, requests.post, path_array, json_body)
         # import logging, json
         # logging.debug(f"resp (make_request) = \n{json.dumps(resp, indent=2)}")
         request_id = resp['id']
-        timeout_minutes = 2
         is_completed = False
         is_failed = False
         try_count = -1
-        delay = 5  # seconds
+        delay = 15  # seconds
         max_retries = (timeout_minutes * 60) // delay
         # wait 1 cycle before start polling
         time.sleep(delay)
@@ -741,7 +815,7 @@ def make_service_post_request(self, config: SolaceTaskBrokerConfig, path_array:
                 resp, resp, self.get_module()._name, module_op)
         if not is_completed:
             msg = [
-                f"timeout service post request - not completed, state={resp['adminProgress']}", str(resp)]
+                f"timeout service post request - not completed, timeout(mins)={timeout_minutes}, state={resp['adminProgress']}", str(resp)]
             raise SolaceInternalError(msg)
         return resp
 
@@ -794,7 +868,7 @@ def filter(self, settings: dict, query_params: dict) -> dict:
 
     def get_cert_authority(self, config, service_id, cert_authority_name, query_params):
         # GET services/{serviceId}/serviceCertificateAuthorities/{certAuthorityName}
-        path_array = [SolaceCloudApi.API_BASE_PATH, SolaceCloudApi.API_SERVICES,
+        path_array = [self.get_api_base_path(config), SolaceCloudApi.API_SERVICES,
                       service_id, 'serviceCertificateAuthorities', cert_authority_name]
         resp = self.get_object_settings(config, path_array)
         cert_authority = resp['certificate']

src/ansible_collections/solace/pubsub_plus/plugins/module_utils/solace_error.py

@@ -77,6 +77,11 @@ def __init__(self, param, value, msg):
         super().__init__(f"arg '{param}={value}': {msg}")
 
 
+class SolaceEnvVarError(Exception):
+    def __init__(self, name, value, msg):
+        super().__init__(f"invalid env var: '{name}={value}'. {msg}.")
+
+
 class SolaceFeatureNotSupportedError(Exception):
     def __init__(self, feature):
         super().__init__(f"feature: '{feature}")