diff --git a/src/azure-cli/azure/cli/command_modules/storage/_params.py b/src/azure-cli/azure/cli/command_modules/storage/_params.py index 7daa2377abc..62d648f8ba7 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/_params.py +++ b/src/azure-cli/azure/cli/command_modules/storage/_params.py @@ -2417,7 +2417,7 @@ def load_arguments(self, _): # pylint: disable=too-many-locals, too-many-statem help='Show nextMarker in result when specified.') for item in ['create', 'show', 'delete', 'exists', 'upload', 'append', 'download', 'show', 'metadata update', - 'metadata show']: + 'metadata show', 'set-expiry']: with self.argument_context('storage fs file {}'.format(item)) as c: c.extra('file_system_name', options_list=['-f', '--file-system'], help='File system name (i.e. container name).', required=True) @@ -2473,6 +2473,16 @@ def load_arguments(self, _): # pylint: disable=too-many-locals, too-many-statem c.argument('permissions', permissions_type) c.argument('umask', umask_type) + with self.argument_context('storage fs file set-expiry') as c: + t_expiry_option_type = self.get_models('_generated.models#PathExpiryOptions', + resource_type=ResourceType.DATA_STORAGE_FILEDATALAKE) + c.argument('expiry_options', required=True, arg_type=get_enum_type(t_expiry_option_type)) + from ._validators import validate_fs_file_set_expiry + c.argument('expires_on', validator=validate_fs_file_set_expiry, + help='The time to set the file to expiry. When expiry_options is ' + 'RelativeTo*, expires_on should be an int in milliseconds. If the ' + 'type of expires_on is datetime, it should be in UTC time.') + for item in ['set', 'show']: with self.argument_context('storage fs access {}'.format(item)) as c: from ._validators import validate_access_control diff --git a/src/azure-cli/azure/cli/command_modules/storage/_validators.py b/src/azure-cli/azure/cli/command_modules/storage/_validators.py index dcf919fb4d4..12c1cd15beb 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/_validators.py +++ b/src/azure-cli/azure/cli/command_modules/storage/_validators.py @@ -2211,3 +2211,10 @@ def validate_blob_arguments(namespace): def encode_deleted_path(namespace): from urllib.parse import quote namespace.deleted_path_name = quote(namespace.deleted_path_name) + + +def validate_fs_file_set_expiry(namespace): + try: + namespace.expires_on = get_datetime_type(False)(namespace.expires_on) + except ValueError: + pass diff --git a/src/azure-cli/azure/cli/command_modules/storage/commands.py b/src/azure-cli/azure/cli/command_modules/storage/commands.py index 7805859c61e..98869964138 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/commands.py +++ b/src/azure-cli/azure/cli/command_modules/storage/commands.py @@ -930,6 +930,7 @@ def get_custom_sdk(custom_module, client_factory, resource_type=ResourceType.DAT g.storage_command_oauth('metadata update', 'set_metadata') g.storage_command_oauth('metadata show', 'get_file_properties', exception_handler=show_exception_handler, transform=transform_metadata) + g.storage_command_oauth('set-expiry', "set_file_expiry") with self.command_group('storage fs access', adls_directory_sdk, custom_command_type=custom_adls_directory_sdk, resource_type=ResourceType.DATA_STORAGE_FILEDATALAKE, min_api='2018-11-09') as g: diff --git a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/recordings/test_adls_file_set_expiry_scenarios.yaml b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/recordings/test_adls_file_set_expiry_scenarios.yaml new file mode 100644 index 00000000000..92fb0adf5b5 --- /dev/null +++ b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/recordings/test_adls_file_set_expiry_scenarios.yaml @@ -0,0 +1,490 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - storage account keys list + Connection: + - keep-alive + Content-Length: + - '0' + ParameterSetName: + - -n -g --query -o + User-Agent: + - AZURECLI/2.38.0 azsdk-python-azure-mgmt-storage/20.0.0 Python/3.9.6 (Windows-10-10.0.19044-SP0) + method: POST + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/clitest.rg000001/providers/Microsoft.Storage/storageAccounts/clitest000002/listKeys?api-version=2021-09-01&$expand=kerb + response: + body: + string: '{"keys":[{"creationTime":"2022-08-01T08:33:30.3037820Z","keyName":"key1","value":"veryFakedStorageAccountKey==","permissions":"FULL"},{"creationTime":"2022-08-01T08:33:30.3037820Z","keyName":"key2","value":"veryFakedStorageAccountKey==","permissions":"FULL"}]}' + headers: + cache-control: + - no-cache + content-length: + - '260' + content-type: + - application/json + date: + - Mon, 01 Aug 2022 08:33:51 GMT + expires: + - '-1' + pragma: + - no-cache + server: + - Microsoft-Azure-Storage-Resource-Provider/1.0,Microsoft-HTTPAPI/2.0 Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-ratelimit-remaining-subscription-resource-requests: + - '11998' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + CommandName: + - storage fs create + Connection: + - keep-alive + Content-Length: + - '0' + ParameterSetName: + - -n --account-name --account-key + User-Agent: + - AZURECLI/2.38.0 azsdk-python-storage-dfs/12.7.0 Python/3.9.6 (Windows-10-10.0.19044-SP0) + x-ms-date: + - Mon, 01 Aug 2022 08:33:52 GMT + x-ms-version: + - '2021-06-08' + method: PUT + uri: https://clitest000002.blob.core.windows.net/filesystem000003?restype=container + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Mon, 01 Aug 2022 08:33:52 GMT + etag: + - '"0x8DA7398912659F7"' + last-modified: + - Mon, 01 Aug 2022 08:33:53 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-version: + - '2021-06-08' + status: + code: 201 + message: Created +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - storage fs file create + Connection: + - keep-alive + Content-Length: + - '0' + ParameterSetName: + - -p -f --content-type --account-name --account-key + User-Agent: + - AZURECLI/2.38.0 azsdk-python-storage-dfs/12.7.0 Python/3.9.6 (Windows-10-10.0.19044-SP0) + x-ms-content-type: + - application/json + x-ms-date: + - Mon, 01 Aug 2022 08:33:53 GMT + x-ms-version: + - '2021-06-08' + method: PUT + uri: https://clitest000002.dfs.core.windows.net/filesystem000003/file000004?resource=file + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Mon, 01 Aug 2022 08:33:54 GMT + etag: + - '"0x8DA739892093B81"' + last-modified: + - Mon, 01 Aug 2022 08:33:54 GMT + server: + - Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0 + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2021-06-08' + status: + code: 201 + message: Created +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - storage fs file set-expiry + Connection: + - keep-alive + Content-Length: + - '0' + ParameterSetName: + - --expiry-options --expires-on -p -f --account-name --account-key + User-Agent: + - AZURECLI/2.38.0 azsdk-python-storage-dfs/12.7.0 Python/3.9.6 (Windows-10-10.0.19044-SP0) + x-ms-date: + - Mon, 01 Aug 2022 08:33:55 GMT + x-ms-expiry-option: + - Absolute + x-ms-expiry-time: + - Mon, 01 Aug 2022 08:34:00 GMT + x-ms-version: + - '2021-06-08' + method: PUT + uri: https://clitest000002.blob.core.windows.net/filesystem000003/file000004?comp=expiry + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Mon, 01 Aug 2022 08:33:55 GMT + etag: + - '"0x8DA739892093B81"' + last-modified: + - Mon, 01 Aug 2022 08:33:54 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-version: + - '2021-06-08' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + CommandName: + - storage fs file exists + Connection: + - keep-alive + ParameterSetName: + - -p -f --account-name --account-key + User-Agent: + - AZURECLI/2.38.0 azsdk-python-storage-dfs/12.7.0 Python/3.9.6 (Windows-10-10.0.19044-SP0) + x-ms-date: + - Mon, 01 Aug 2022 08:33:56 GMT + x-ms-version: + - '2021-06-08' + method: HEAD + uri: https://clitest000002.blob.core.windows.net/filesystem000003/file000004 + response: + body: + string: '' + headers: + accept-ranges: + - bytes + content-length: + - '0' + content-type: + - application/json + date: + - Mon, 01 Aug 2022 08:33:57 GMT + etag: + - '"0x8DA739892093B81"' + last-modified: + - Mon, 01 Aug 2022 08:33:54 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-access-tier: + - Hot + x-ms-access-tier-inferred: + - 'true' + x-ms-blob-type: + - BlockBlob + x-ms-creation-time: + - Mon, 01 Aug 2022 08:33:54 GMT + x-ms-expiry-time: + - Mon, 01 Aug 2022 08:34:00 GMT + x-ms-group: + - $superuser + x-ms-lease-state: + - available + x-ms-lease-status: + - unlocked + x-ms-owner: + - $superuser + x-ms-permissions: + - rw-r----- + x-ms-resource-type: + - file + x-ms-server-encrypted: + - 'true' + x-ms-version: + - '2021-06-08' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + CommandName: + - storage fs file exists + Connection: + - keep-alive + ParameterSetName: + - -p -f --account-name --account-key + User-Agent: + - AZURECLI/2.38.0 azsdk-python-storage-dfs/12.7.0 Python/3.9.6 (Windows-10-10.0.19044-SP0) + x-ms-date: + - Mon, 01 Aug 2022 08:34:04 GMT + x-ms-version: + - '2021-06-08' + method: HEAD + uri: https://clitest000002.blob.core.windows.net/filesystem000003/file000004 + response: + body: + string: '' + headers: + date: + - Mon, 01 Aug 2022 08:34:04 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + transfer-encoding: + - chunked + x-ms-error-code: + - BlobNotFound + x-ms-version: + - '2021-06-08' + status: + code: 404 + message: The specified blob does not exist. +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - storage fs file create + Connection: + - keep-alive + Content-Length: + - '0' + ParameterSetName: + - -p -f --content-type --account-name --account-key + User-Agent: + - AZURECLI/2.38.0 azsdk-python-storage-dfs/12.7.0 Python/3.9.6 (Windows-10-10.0.19044-SP0) + x-ms-content-type: + - application/json + x-ms-date: + - Mon, 01 Aug 2022 08:34:05 GMT + x-ms-version: + - '2021-06-08' + method: PUT + uri: https://clitest000002.dfs.core.windows.net/filesystem000003/file000004?resource=file + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Mon, 01 Aug 2022 08:34:05 GMT + etag: + - '"0x8DA739898F88ED9"' + last-modified: + - Mon, 01 Aug 2022 08:34:06 GMT + server: + - Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0 + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2021-06-08' + status: + code: 201 + message: Created +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - storage fs file set-expiry + Connection: + - keep-alive + Content-Length: + - '0' + ParameterSetName: + - --expiry-options --expires-on -p -f --account-name --account-key + User-Agent: + - AZURECLI/2.38.0 azsdk-python-storage-dfs/12.7.0 Python/3.9.6 (Windows-10-10.0.19044-SP0) + x-ms-date: + - Mon, 01 Aug 2022 08:34:06 GMT + x-ms-expiry-option: + - RelativeToCreation + x-ms-expiry-time: + - '5000' + x-ms-version: + - '2021-06-08' + method: PUT + uri: https://clitest000002.blob.core.windows.net/filesystem000003/file000004?comp=expiry + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Mon, 01 Aug 2022 08:34:07 GMT + etag: + - '"0x8DA739898F88ED9"' + last-modified: + - Mon, 01 Aug 2022 08:34:06 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-version: + - '2021-06-08' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + CommandName: + - storage fs file exists + Connection: + - keep-alive + ParameterSetName: + - -p -f --account-name --account-key + User-Agent: + - AZURECLI/2.38.0 azsdk-python-storage-dfs/12.7.0 Python/3.9.6 (Windows-10-10.0.19044-SP0) + x-ms-date: + - Mon, 01 Aug 2022 08:34:07 GMT + x-ms-version: + - '2021-06-08' + method: HEAD + uri: https://clitest000002.blob.core.windows.net/filesystem000003/file000004 + response: + body: + string: '' + headers: + accept-ranges: + - bytes + content-length: + - '0' + content-type: + - application/json + date: + - Mon, 01 Aug 2022 08:34:08 GMT + etag: + - '"0x8DA739898F88ED9"' + last-modified: + - Mon, 01 Aug 2022 08:34:06 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-access-tier: + - Hot + x-ms-access-tier-inferred: + - 'true' + x-ms-blob-type: + - BlockBlob + x-ms-creation-time: + - Mon, 01 Aug 2022 08:34:06 GMT + x-ms-expiry-time: + - Mon, 01 Aug 2022 08:34:11 GMT + x-ms-group: + - $superuser + x-ms-lease-state: + - available + x-ms-lease-status: + - unlocked + x-ms-owner: + - $superuser + x-ms-permissions: + - rw-r----- + x-ms-resource-type: + - file + x-ms-server-encrypted: + - 'true' + x-ms-version: + - '2021-06-08' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + CommandName: + - storage fs file exists + Connection: + - keep-alive + ParameterSetName: + - -p -f --account-name --account-key + User-Agent: + - AZURECLI/2.38.0 azsdk-python-storage-dfs/12.7.0 Python/3.9.6 (Windows-10-10.0.19044-SP0) + x-ms-date: + - Mon, 01 Aug 2022 08:34:16 GMT + x-ms-version: + - '2021-06-08' + method: HEAD + uri: https://clitest000002.blob.core.windows.net/filesystem000003/file000004 + response: + body: + string: '' + headers: + date: + - Mon, 01 Aug 2022 08:34:16 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + transfer-encoding: + - chunked + x-ms-error-code: + - BlobNotFound + x-ms-version: + - '2021-06-08' + status: + code: 404 + message: The specified blob does not exist. +version: 1 diff --git a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_adls_gen2_scenarios.py b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_adls_gen2_scenarios.py index 7a134720a61..a5faa4fe91c 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_adls_gen2_scenarios.py +++ b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_adls_gen2_scenarios.py @@ -498,6 +498,36 @@ def test_adls_file_scenarios(self, resource_group, storage_account): self.storage_cmd('storage fs file exists -p {} -f {}', account_info, file, new_filesystem) \ .assert_with_checks(JMESPathCheck('exists', False)) + @ResourceGroupPreparer() + @StorageAccountPreparer(kind="StorageV2", hns=True) + def test_adls_file_set_expiry_scenarios(self, resource_group, storage_account): + from datetime import datetime, timedelta + account_info = self.get_account_info(resource_group, storage_account) + filesystem = self.create_file_system(account_info) + file = self.create_random_name(prefix="file", length=12) + + self.storage_cmd('storage fs file create -p {} -f {} --content-type "application/json"', + account_info, file, filesystem) + expiry = (datetime.utcnow() + timedelta(seconds=5)).strftime('%Y-%m-%dT%H:%M:%SZ') + self.storage_cmd("storage fs file set-expiry --expiry-options Absolute --expires-on {} -p {} -f {}", + account_info, expiry, file, filesystem) + self.storage_cmd("storage fs file exists -p {} -f {}", account_info, file, filesystem)\ + .assert_with_checks(JMESPathCheck('exists', True)) + time.sleep(7) + self.storage_cmd("storage fs file exists -p {} -f {}", account_info, file, filesystem)\ + .assert_with_checks(JMESPathCheck('exists', False)) + + self.storage_cmd('storage fs file create -p {} -f {} --content-type "application/json"', + account_info, file, filesystem) + expiry = 5000 + self.storage_cmd("storage fs file set-expiry --expiry-options RelativeToCreation --expires-on {} -p {} -f {}", + account_info, expiry, file, filesystem) + self.storage_cmd("storage fs file exists -p {} -f {}", account_info, file, filesystem) \ + .assert_with_checks(JMESPathCheck('exists', True)) + time.sleep(7) + self.storage_cmd("storage fs file exists -p {} -f {}", account_info, file, filesystem) \ + .assert_with_checks(JMESPathCheck('exists', False)) + @ResourceGroupPreparer() @StorageAccountPreparer(kind="StorageV2", hns=True) def test_adls_metadata_scenarios(self, resource_group, storage_account):