From 2b5a980cd03bed195a1443fa1d0d81a8a06ff36c Mon Sep 17 00:00:00 2001 From: Gauri Prasad Date: Thu, 20 Jun 2024 11:45:27 -0700 Subject: [PATCH 1/2] Migrated smoke tests for auto detect blob type --- .../zt_newe2e_autodetect_blob_type_test.go | 109 +++++++++++++++++ testSuite/scripts/run.py | 2 - .../scripts/test_autodetect_blob_type.py | 110 ------------------ 3 files changed, 109 insertions(+), 112 deletions(-) create mode 100644 e2etest/zt_newe2e_autodetect_blob_type_test.go delete mode 100644 testSuite/scripts/test_autodetect_blob_type.py diff --git a/e2etest/zt_newe2e_autodetect_blob_type_test.go b/e2etest/zt_newe2e_autodetect_blob_type_test.go new file mode 100644 index 000000000..190e9f19c --- /dev/null +++ b/e2etest/zt_newe2e_autodetect_blob_type_test.go @@ -0,0 +1,109 @@ +package e2etest + +import ( + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-storage-azcopy/v10/common" +) + +func init() { + suiteManager.RegisterSuite(&AutoDetectBlobTypeTestSuite{}) +} + +type AutoDetectBlobTypeTestSuite struct{} + +func (s *AutoDetectBlobTypeTestSuite) Scenario_AutoInferBlobTypeVHD(svm *ScenarioVariationManager) { + fileName := "myVHD.vHd" // awkward capitalization to see if AzCopy catches it. + body := NewRandomObjectContentContainer(svm, 4*common.MegaByte) + + srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). + GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + dstObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}). + GetObject(svm, fileName, common.EEntityType.File()) + + // copy vhd file without specifying page blob. Page blob is inferred for VHD, VHDX, and VMDK + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObj, dstObj}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + BlockSizeMB: pointerTo(4.0), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{ + Body: body, + ObjectProperties: ObjectProperties{ + BlobProperties: BlobProperties{ + Type: pointerTo(blob.BlobTypePageBlob), + }, + }, + }, true) +} + +func (s *AutoDetectBlobTypeTestSuite) Scenario_InferBlobTypeFilePageBlob(svm *ScenarioVariationManager) { + fileName := "testS2SVHD.vhd" + body := NewRandomObjectContentContainer(svm, 4*common.MegaByte) + + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{}) + srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcContainer, dstContainer}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ + Body: body, + ObjectProperties: ObjectProperties{ + BlobProperties: BlobProperties{ + Type: pointerTo(blob.BlobTypePageBlob), + }, + }, + }, true) +} + +func (s *AutoDetectBlobTypeTestSuite) Scenario_DetectBlobTypeBlobBlob(svm *ScenarioVariationManager) { + fileName := "testS2SVHD.vhd" + body := NewRandomObjectContentContainer(svm, 4*common.MegaByte) + + // Upload to Azure Blob Storage as Block Blob and detect as Block Blob. + // AzCopy detects the source blob type. + // This means that in all scenarios EXCEPT Blob -> Blob, .vhd corresponds to page blob. + // However, in Blob -> Blob, we preserve the blob type instead of detecting it. + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) + srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcContainer, dstContainer}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ + Body: body, + ObjectProperties: ObjectProperties{ + BlobProperties: BlobProperties{ + Type: pointerTo(blob.BlobTypeBlockBlob), + }, + }, + }, true) +} diff --git a/testSuite/scripts/run.py b/testSuite/scripts/run.py index 899b8e37e..6184a79ac 100644 --- a/testSuite/scripts/run.py +++ b/testSuite/scripts/run.py @@ -14,7 +14,6 @@ from test_google_cloud_storage_copy import * from test_blobfs_download_SAS import * from test_blobfs_upload_SAS import * -from test_autodetect_blob_type import * from test_file_sync import * from test_file_copy import * from test_google_cloud_storage_copy import * @@ -228,7 +227,6 @@ def main(): BlobFs_Upload_ShareKey_User_Scenarios, BlobFs_Download_SharedKey_User_Scenarios, Service_2_Service_Copy_User_Scenario, - Autodetect_Blob_Type_Scenario, Google_Cloud_Storage_Copy_User_Scenario] suites_list = [] diff --git a/testSuite/scripts/test_autodetect_blob_type.py b/testSuite/scripts/test_autodetect_blob_type.py deleted file mode 100644 index afb6e1338..000000000 --- a/testSuite/scripts/test_autodetect_blob_type.py +++ /dev/null @@ -1,110 +0,0 @@ -import filecmp -import os -import unittest -import utility as util - -class Autodetect_Blob_Type_Scenario(unittest.TestCase): - def setUp(self): - cmd = util.Command("login").add_arguments("--service-principal").add_flags("application-id", os.environ['ACTIVE_DIRECTORY_APPLICATION_ID']).add_flags("tenant-id", os.environ['OAUTH_TENANT_ID']) - cmd.execute_azcopy_copy_command() - - def tearDown(self): - cmd = util.Command("logout") - cmd.execute_azcopy_copy_command() - - # Currently, the only auto-detected blob type is page blob. - # Copy a VHD without specifying page blob and see what it does. - def test_auto_infer_blob_type_vhd(self): - # Upload to blob storage, detecting as a page blob. - file_name = "myVHD.vHd" # awkward capitalization to see if azcopy catches it - file_path = util.create_test_file(file_name, 4 * 1024 * 1024) - - # copy VHD file without specifying page blob. Page blob is inferred for VHD, VHDX, and VMDK - destination_sas = util.get_object_sas(util.test_container_url, file_name) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("log-level", - "info"). \ - add_flags("block-size-mb", 4).execute_azcopy_copy_command() - self.assertTrue(result) - - # execute validator. Validator will ensure it's a page blob as validator now checks blob type before testing. - result = util.Command("testBlob").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("blob-type", "PageBlob").execute_azcopy_verify() - self.assertTrue(result) - - def test_copy_infer_blob_type_from_files_to_page_blob(self): - # Upload to Azure Files and then transfer to Blob to detect as page blob. - file_name = "testS2SVHD.vhd" - containerName = util.get_resource_name("s2sbtautodetect") - - # These run on separate accounts in CI, so even without "dst", it's OK. - # Needed this to run on a single account, though. - dstbase = util.get_object_sas(util.test_s2s_dst_blob_account_url, containerName + "dst") - srcbase = util.get_object_sas(util.test_s2s_src_file_account_url, containerName) - - result = util.Command("create").add_arguments(srcbase).add_flags("serviceType", "File"). \ - add_flags("resourceType", "Bucket").execute_azcopy_create() - self.assertTrue(result) - - result = util.Command("create").add_arguments(dstbase).add_flags("serviceType", "Blob"). \ - add_flags("resourceType", "Bucket").execute_azcopy_create() - self.assertTrue(result) - - src_container_url = util.get_object_sas(srcbase, file_name) - dst_container_url = util.get_object_sas(dstbase, file_name) - - file_path = util.create_test_file(file_name, 4 * 1024 * 1024) - - result = util.Command("copy").add_arguments(file_path).add_arguments(src_container_url). \ - add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - result = util.Command("copy").add_arguments(src_container_url).add_arguments(dst_container_url). \ - add_flags("log-level", "info").execute_azcopy_copy_command() # Blob type in this case will be inferred. - self.assertTrue(result) - - # Verify blob type (should be page blob) - result = util.Command("testBlob").add_arguments(file_path).add_arguments(dst_container_url). \ - add_flags("blob-type", "PageBlob").execute_azcopy_verify() - self.assertTrue(result) - - def test_copy_detect_blob_type_from_blob_to_blob(self): - # Upload to Azure Blob Storage as Block Blob and detect as Block Blob. - # Why is this necessary, you ask? - # Because we detect the source blob type. - # This means that in all scenarios EXCEPT Blob -> Blob, .vhd corresponds to page blob. - # However, in Blob -> Blob, we preserve the blob type instead of detecting it. - # As a result, this behavior should be tested. - file_name = "testS2SVHD.vhd" - containerName = util.get_resource_name("s2sbtautodetect") - - # These run on separate accounts in CI, so even without "dst", it's OK. - # Needed this to run on a single account, though. - dstbase = util.get_object_sas(util.test_s2s_dst_blob_account_url, containerName + "dst") - srcbase = util.get_object_sas(util.test_s2s_src_blob_account_url, containerName) - - result = util.Command("create").add_arguments(srcbase).add_flags("serviceType", "Blob"). \ - add_flags("resourceType", "Bucket").execute_azcopy_create() - self.assertTrue(result) - - result = util.Command("create").add_arguments(dstbase).add_flags("serviceType", "Blob"). \ - add_flags("resourceType", "Bucket").execute_azcopy_create() - self.assertTrue(result) - - src_container_url = util.get_object_sas(srcbase, file_name) - dst_container_url = util.get_object_sas(dstbase, file_name) - - file_path = util.create_test_file(file_name, 4 * 1024 * 1024) - - # Explicitly define BlockBlob as this is local -> blob - result = util.Command("copy").add_arguments(file_path).add_arguments(src_container_url). \ - add_flags("blob-type", "BlockBlob").add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - result = util.Command("copy").add_arguments(src_container_url).add_arguments(dst_container_url). \ - add_flags("log-level", "info").execute_azcopy_copy_command() # Blob type will be detected as block. - self.assertTrue(result) - - # Verify blob type (should be block blob) - result = util.Command("testBlob").add_arguments(file_path).add_arguments(dst_container_url). \ - add_flags("blob-type", "BlockBlob").execute_azcopy_verify() - self.assertTrue(result) From 08e6f7f053f3f388fd291cea39096fb6a3ddab57 Mon Sep 17 00:00:00 2001 From: Gauri Prasad Date: Tue, 16 Jul 2024 13:55:05 -0700 Subject: [PATCH 2/2] combine tests --- .../zt_newe2e_autodetect_blob_type_test.go | 70 +------------------ 1 file changed, 3 insertions(+), 67 deletions(-) diff --git a/e2etest/zt_newe2e_autodetect_blob_type_test.go b/e2etest/zt_newe2e_autodetect_blob_type_test.go index 190e9f19c..1762209e1 100644 --- a/e2etest/zt_newe2e_autodetect_blob_type_test.go +++ b/e2etest/zt_newe2e_autodetect_blob_type_test.go @@ -11,11 +11,11 @@ func init() { type AutoDetectBlobTypeTestSuite struct{} -func (s *AutoDetectBlobTypeTestSuite) Scenario_AutoInferBlobTypeVHD(svm *ScenarioVariationManager) { +func (s *AutoDetectBlobTypeTestSuite) Scenario_AutoInferDetectBlobTypeVHD(svm *ScenarioVariationManager) { fileName := "myVHD.vHd" // awkward capitalization to see if AzCopy catches it. body := NewRandomObjectContentContainer(svm, 4*common.MegaByte) - srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). + srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, ResolveVariation(svm, []common.Location{common.ELocation.Local(), common.ELocation.File(), common.ELocation.Blob()})), ResourceDefinitionContainer{}). GetObject(svm, fileName, common.EEntityType.File()) srcObj.Create(svm, body, ObjectProperties{}) @@ -38,71 +38,7 @@ func (s *AutoDetectBlobTypeTestSuite) Scenario_AutoInferBlobTypeVHD(svm *Scenari Body: body, ObjectProperties: ObjectProperties{ BlobProperties: BlobProperties{ - Type: pointerTo(blob.BlobTypePageBlob), - }, - }, - }, true) -} - -func (s *AutoDetectBlobTypeTestSuite) Scenario_InferBlobTypeFilePageBlob(svm *ScenarioVariationManager) { - fileName := "testS2SVHD.vhd" - body := NewRandomObjectContentContainer(svm, 4*common.MegaByte) - - srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{}) - srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) - srcObj.Create(svm, body, ObjectProperties{}) - - dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) - - RunAzCopy(svm, AzCopyCommand{ - Verb: AzCopyVerbCopy, - Targets: []ResourceManager{srcContainer, dstContainer}, - Flags: CopyFlags{ - CopySyncCommonFlags: CopySyncCommonFlags{ - Recursive: pointerTo(true), - }, - }, - }) - - ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ - Body: body, - ObjectProperties: ObjectProperties{ - BlobProperties: BlobProperties{ - Type: pointerTo(blob.BlobTypePageBlob), - }, - }, - }, true) -} - -func (s *AutoDetectBlobTypeTestSuite) Scenario_DetectBlobTypeBlobBlob(svm *ScenarioVariationManager) { - fileName := "testS2SVHD.vhd" - body := NewRandomObjectContentContainer(svm, 4*common.MegaByte) - - // Upload to Azure Blob Storage as Block Blob and detect as Block Blob. - // AzCopy detects the source blob type. - // This means that in all scenarios EXCEPT Blob -> Blob, .vhd corresponds to page blob. - // However, in Blob -> Blob, we preserve the blob type instead of detecting it. - srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) - srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) - srcObj.Create(svm, body, ObjectProperties{}) - - dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) - - RunAzCopy(svm, AzCopyCommand{ - Verb: AzCopyVerbCopy, - Targets: []ResourceManager{srcContainer, dstContainer}, - Flags: CopyFlags{ - CopySyncCommonFlags: CopySyncCommonFlags{ - Recursive: pointerTo(true), - }, - }, - }) - - ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ - Body: body, - ObjectProperties: ObjectProperties{ - BlobProperties: BlobProperties{ - Type: pointerTo(blob.BlobTypeBlockBlob), + Type: common.Iff(srcObj.Location() == common.ELocation.Blob(), pointerTo(blob.BlobTypeBlockBlob), pointerTo(blob.BlobTypePageBlob)), }, }, }, true)