Changes for 0.8.9 (uamqp focus)

.azure-devops/merge.yml

@@ -89,7 +89,7 @@ jobs:
 - job: 'Run_Tests_Mac'
   dependsOn: ['Build_Publish_Azure_CLI_Test_SDK','Build_Publish_Azure_IoT_CLI_Extension']
   pool:
-    vmImage: 'macOS-10.13'
+    vmImage: 'macOS-10.14'
 
   steps:
   - template: templates/run-tests.yml

.azure-devops/templates/install-azure-cli-edge.yml

@@ -1,4 +1,3 @@
 steps:
   - script: 'pip install --pre azure-cli --extra-index-url https://azurecliprod.blob.core.windows.net/edge'
     displayName: 'Install Azure CLI edge'
-

.azure-devops/templates/install-azure-cli-released.yml

@@ -1,4 +1,3 @@
-
 steps:
   - script: pip install azure-cli
-    displayName: 'Install Azure CLI released'
+    displayName: 'Install Azure CLI released'

.azure-devops/templates/run-tests.yml

@@ -19,34 +19,37 @@ steps:
 
   - template: setup-ci-machine.yml
 
-  - template: download-install-local-azure-iot-cli-extension-with-pip.yml
+  - template: download-install-local-azure-iot-cli-extension.yml
+
+  - template: set-pythonpath.yml
 
   - ${{ if eq(parameters.runUnitTestsOnly, 'false') }}:
     - script: pytest --junitxml "TEST-results.xml"
       displayName: 'Execute all Tests'
-  
+
   - ${{ if eq(parameters.runUnitTestsOnly, 'true') }}:
-    - script: pytest -v azext_iot/tests/test_iot_ext_unit.py --junitxml "TEST-iothub-unit-results.xml"
+    - script: pytest -v azext_iot/tests/test_iot_ext_unit.py --junitxml=junit/test-iothub-unit-results.xml
       displayName: 'Execute IoT Hub unit tests'
-    - script: pytest -v azext_iot/tests/test_iot_dps_unit.py --junitxml "TEST-dps-unit-results.xml"
+    - script: pytest -v azext_iot/tests/test_iot_dps_unit.py --junitxml=junit/test-dps-unit-results.xml
       displayName: 'Execute DPS unit tests'
-    - script: pytest -v azext_iot/tests/test_iot_utility_unit.py --junitxml "TEST-utility-unit-results.xml"
+    - script: pytest -v azext_iot/tests/test_iot_utility_unit.py --junitxml=junit/test-utility-unit-results.xml
       displayName: 'Execute Utility unit tests'
-    - script: pytest -v azext_iot/tests/test_iot_central_unit.py --junitxml "TEST-central-unit-results.xml"
+    - script: pytest -v azext_iot/tests/test_iot_central_unit.py --junitxml=junit/test-central-unit-results.xml
       displayName: 'Execute IoT Central unit tests'
-    - script: pytest -v azext_iot/tests/test_iot_pnp_unit.py --junitxml "TEST-pnp-unit-results.xml"
+    - script: pytest -v azext_iot/tests/test_iot_pnp_unit.py --junitxml=junit/test-pnp-unit-results.xml
       displayName: 'Execute IoT PnP unit tests'
-    - script: pytest -v azext_iot/tests/test_iot_digitaltwin_unit.py --junitxml "TEST-dt-unit-results.xml"
+    - script: pytest -v azext_iot/tests/test_iot_digitaltwin_unit.py --junitxml=junit/test-dt-unit-results.xml
       displayName: 'Execute IoT DigitalTwin unit tests'
-    - script: pytest -v azext_iot/tests/configurations/test_iot_config_unit.py --junitxml "TEST-config-unit-results.xml"
+    - script: pytest -v azext_iot/tests/configurations/test_iot_config_unit.py --junitxml=junit/test-config-unit-results.xml
       displayName: 'Execute IoT Configuration unit tests'
-    - script: pytest -v azext_iot/tests/jobs/test_iothub_jobs_unit.py --junitxml "TEST-jobs-unit-results.xml"
+    - script: pytest -v azext_iot/tests/jobs/test_iothub_jobs_unit.py --junitxml=junit/test-jobs-unit-results.xml
       displayName: 'Execute IoT Hub job unit tests'
 
   - task: PublishTestResults@2
+    condition: succeededOrFailed()
     displayName: 'Publish Test Results'
     inputs:
       testResultsFormat: 'JUnit'
-      testResultsFiles: '**TEST-*.xml'
+      testResultsFiles: '**/test-*.xml'
+      testRunTitle: 'Publish test results for Python ${{ parameters.pythonVersion }} on OS $(Agent.OS)'
       searchFolder: '$(System.DefaultWorkingDirectory)'
-    condition: succeededOrFailed()

.azure-devops/templates/set-pythonpath.yml

@@ -0,0 +1,12 @@
+steps:
+  - task: PythonScript@0
+    displayName : 'Extract extension path'
+    name: 'extractExtensionPath'
+    inputs:
+      scriptSource: 'inline'
+      script: |
+        from azure.cli.core.extension import get_extension_path
+        from six import print_
+        extension_path = get_extension_path("azure-cli-iot-ext")
+        print_("Extension path is " + extension_path)
+        print_("##vso[task.setvariable variable=PYTHONPATH;]"+extension_path)

azext_iot/_factory.py

@@ -25,7 +25,13 @@ def iot_hub_service_factory(cli_ctx, *_):
             working with IoT Hub.
     """
     from azure.cli.core.commands.client_factory import get_mgmt_service_client
-    from azure.mgmt.iothub.iot_hub_client import IotHubClient
+
+    # To support newer and older IotHubClient. 0.9.0+ has breaking changes.
+    try:
+        from azure.mgmt.iothub import IotHubClient
+    except:
+        # For <0.9.0
+        from azure.mgmt.iothub.iot_hub_client import IotHubClient
     return get_mgmt_service_client(cli_ctx, IotHubClient).iot_hub_resource
 
 

azext_iot/_validators.py

@@ -13,18 +13,18 @@ def mode2_iot_login_handler(cmd, namespace):
         args = vars(namespace)
         arg_keys = args.keys()
         if 'login' in arg_keys:
-            login_value = args.get('login')
+            login_value = args['login']
             iot_cmd_type = None
             entity_value = None
             if 'hub_name' in arg_keys:
                 iot_cmd_type = 'IoT Hub'
-                entity_value = args.get('hub_name')
+                entity_value = args['hub_name']
             elif 'dps_name' in arg_keys:
                 iot_cmd_type = 'DPS'
-                entity_value = args.get('dps_name')
+                entity_value = args['dps_name']
             elif 'repo_endpoint' in arg_keys:
                 iot_cmd_type = 'PnP'
-                entity_value = args.get('repo_endpoint')
+                entity_value = args['repo_endpoint']
 
             if not any([login_value, entity_value]):
                 raise CLIError(error_no_hub_or_login_on_input(iot_cmd_type))

azext_iot/common/deps.py

@@ -8,6 +8,7 @@
 from os import linesep
 import six
 from six.moves import input
+from knack.util import CLIError
 from azext_iot.constants import EVENT_LIB, VERSION
 from azext_iot.common.utility import test_import
 from azext_iot.common.config import get_uamqp_ext_version, update_uamqp_ext_version
@@ -18,18 +19,19 @@
 def ensure_uamqp(config, yes=False, repair=False):
     if get_uamqp_ext_version(config) != EVENT_LIB[1] or repair or not test_import(EVENT_LIB[0]):
         if not yes:
-            input_txt = ('Dependency update required for IoT extension version: {}. {}'
-                         'Updated dependency must be compatible with {} {}. '
-                         'Continue? (y/n) -> ').format(VERSION, linesep, EVENT_LIB[0], EVENT_LIB[1])
+            input_txt = ('Dependency update ({} {}) required for IoT extension version: {}. {}'
+                         'Continue? (y/n) -> ').format(EVENT_LIB[0], EVENT_LIB[1], VERSION, linesep)
             i = input(input_txt)
             if i.lower() != 'y':
                 sys.exit('User has declined update...')
 
         six.print_('Updating required dependency...')
         with HomebrewPipPatch():
             # The version range defined in this custom_version parameter should be stable
-            if install(EVENT_LIB[0], custom_version='>={},<{}'.format(EVENT_LIB[1], EVENT_LIB[2])):
+            try:
+                install(EVENT_LIB[0], compatible_version='{}'.format(EVENT_LIB[1]))
                 update_uamqp_ext_version(config, EVENT_LIB[1])
-                six.print_('Update appears to have worked. Executing command...')
-            else:
-                sys.exit('Failure updating {} {}. Aborting...'.format(EVENT_LIB[0], EVENT_LIB[1]))
+                six.print_('Update complete. Executing command...')
+            except RuntimeError as e:
+                six.print_('Failure updating {}. Aborting...'.format(EVENT_LIB[0]))
+                raise CLIError(e)

azext_iot/common/pip.py

@@ -39,4 +39,4 @@ def install(package, exact_version=None, compatible_version=None, custom_version
     except subprocess.CalledProcessError as e:
         logger.debug(e.output)
         logger.debug(e)
-        return False
+        raise(RuntimeError(e.output))

azext_iot/constants.py

@@ -7,7 +7,7 @@
 
 import os
 
-VERSION = "0.8.8"
+VERSION = "0.8.9"
 EXTENSION_NAME = "azure-cli-iot-ext"
 EXTENSION_ROOT = os.path.dirname(os.path.abspath(__file__))
 EXTENSION_CONFIG_ROOT_KEY = "iotext"
@@ -44,7 +44,7 @@
 USER_AGENT = "IoTPlatformCliExtension/{}".format(VERSION)
 
 # (Lib name, minimum version (including), maximum version (excluding))
-EVENT_LIB = ("uamqp", "1.1.0", "1.1.1")
+EVENT_LIB = ("uamqp", "1.2", "1.3")
 
 # Config Key's
 CONFIG_KEY_UAMQP_EXT_VERSION = "uamqp_ext_version"

azext_iot/operations/events3/_builders.py

@@ -4,7 +4,8 @@
 from azext_iot.common.sas_token_auth import SasTokenAuthentication
 from azext_iot.common.utility import (parse_entity, unicode_binary_map, url_encode_str)
 
-DEBUG = True
+# To provide amqp frame trace
+DEBUG = False
 
 
 class AmqpBuilder():

azext_iot/operations/events3/_events.py

@@ -18,8 +18,8 @@
 from azext_iot.common.utility import parse_entity, unicode_binary_map, process_json_arg
 from azext_iot.operations.events3._builders import AmqpBuilder
 
-
-DEBUG = True
+# To provide amqp frame trace
+DEBUG = False
 logger = get_logger(__name__)
 
 

azext_iot/tests/__init__.py

@@ -65,6 +65,7 @@ def __init__(self, test_scenario, entity_name, entity_rg, entity_cs):
 
         os.environ["AZURE_CORE_COLLECT_TELEMETRY"] = "no"
         super(IoTLiveScenarioTest, self).__init__(test_scenario)
+        self.region = self.get_region()
 
     def generate_device_names(self, count=1, edge=False):
         names = [
@@ -144,6 +145,15 @@ def tearDown(self):
                     checks=self.is_empty(),
                 )
 
+    def get_region(self):
+        result = self.cmd(
+            "iot hub show -n {}".format(self.entity_name)
+        ).get_output_in_json()
+        locations_set = result["properties"]["locations"]
+        for loc in locations_set:
+            if loc["role"] == "primary":
+                return loc["location"]
+
 
 def disable_telemetry(test_function):
     def wrapper(*args, **kwargs):

azext_iot/tests/settings.py

@@ -17,15 +17,22 @@ class Setting(object):
 # Example of a dynamic class
 # TODO: Evaluate moving this to the extension prime time
 class DynamoSettings(object):
-    def __init__(self, env_set):
-        if not isinstance(env_set, list):
-            raise TypeError("env_set must be a list")
+    def __init__(self, req_env_set, opt_env_set=None):
+        if not isinstance(req_env_set, list):
+            raise TypeError("req_env_set must be a list")
+
         self.env = Setting()
-        self._build_config(env_set)
+        self._build_config(req_env_set)
+
+        if opt_env_set:
+            if not isinstance(opt_env_set, list):
+                raise TypeError("opt_env_set must be a list")
+            self._build_config(opt_env_set, optional=True)
 
-    def _build_config(self, env_set):
+    def _build_config(self, env_set, optional=False):
         for key in env_set:
             value = environ.get(key)
             if not value:
-                raise RuntimeError("'{}' environment variable required.")
+                if not optional:
+                    raise RuntimeError("'{}' environment variable required.".format(key))
             setattr(self.env, key, value)

azext_iot/tests/test_iot_ext_int.py

@@ -6,28 +6,27 @@
 
 import os
 import pytest
+import warnings
 
 from azext_iot.common.utility import read_file_content
 from . import IoTLiveScenarioTest
+from .settings import DynamoSettings, ENV_SET_TEST_IOTHUB_BASIC
 from azext_iot.constants import DEVICE_DEVICESCOPE_PREFIX
 
+opt_env_set = ["azext_iot_teststorageuri"]
 
-# Set these to the proper IoT Hub, IoT Hub Cstring and Resource Group for Live Integration Tests.
-LIVE_HUB = os.environ.get("azext_iot_testhub")
-LIVE_RG = os.environ.get("azext_iot_testrg")
-LIVE_HUB_CS = os.environ.get("azext_iot_testhub_cs")
+settings = DynamoSettings(req_env_set=ENV_SET_TEST_IOTHUB_BASIC, opt_env_set=opt_env_set)
+
+LIVE_HUB = settings.env.azext_iot_testhub
+LIVE_RG = settings.env.azext_iot_testrg
+LIVE_HUB_CS = settings.env.azext_iot_testhub_cs
 LIVE_HUB_MIXED_CASE_CS = LIVE_HUB_CS.replace("HostName", "hostname", 1)
 
 # Set this environment variable to your empty blob container sas uri to test device export and enable file upload test.
 # For file upload, you will need to have configured your IoT Hub before running.
-LIVE_STORAGE = os.environ.get("azext_iot_teststorageuri")
+LIVE_STORAGE = settings.env.azext_iot_teststorageuri
 LIVE_CONSUMER_GROUPS = ["test1", "test2", "test3"]
 
-if not all([LIVE_HUB, LIVE_HUB_CS, LIVE_RG]):
-    raise ValueError(
-        "Set azext_iot_testhub, azext_iot_testhub_cs and azext_iot_testrg to run IoT Hub integration tests."
-    )
-
 CWD = os.path.dirname(os.path.abspath(__file__))
 
 PRIMARY_THUMBPRINT = "A361EA6A7119A8B0B7BBFFA2EAFDAD1F9D5BED8C"
@@ -606,23 +605,26 @@ def test_hub_device_twins(self):
             ],
         )
 
-        # TODO move distributed tracing tests
-        self.cmd(
-            "iot hub distributed-tracing show -d {} -n {} -g {}".format(
-                device_ids[2], LIVE_HUB, LIVE_RG
-            ),
-            checks=self.is_empty(),
-        )
-
-        result = self.cmd(
-            "iot hub distributed-tracing update -d {} -n {} -g {} --sm on --sr 50".format(
-                device_ids[2], LIVE_HUB, LIVE_RG
+        # Region specific test
+        if self.region not in ["West US 2", "North Europe", "Southeast Asia"]:
+            warnings.warn("Skipping distributed-tracing tests. IoT Hub not in supported region!")
+        else:
+            self.cmd(
+                "iot hub distributed-tracing show -d {} -n {} -g {}".format(
+                    device_ids[2], LIVE_HUB, LIVE_RG
+                ),
+                checks=self.is_empty(),
             )
-        ).get_output_in_json()
-        assert result["deviceId"] == device_ids[2]
-        assert result["samplingMode"] == "enabled"
-        assert result["samplingRate"] == "50%"
-        assert not result["isSynced"]
+
+            result = self.cmd(
+                "iot hub distributed-tracing update -d {} -n {} -g {} --sm on --sr 50".format(
+                    device_ids[2], LIVE_HUB, LIVE_RG
+                )
+            ).get_output_in_json()
+            assert result["deviceId"] == device_ids[2]
+            assert result["samplingMode"] == "enabled"
+            assert result["samplingRate"] == "50%"
+            assert not result["isSynced"]
 
 
 class TestIoTHubModules(IoTLiveScenarioTest):