From 2b5a980cd03bed195a1443fa1d0d81a8a06ff36c Mon Sep 17 00:00:00 2001 From: Gauri Prasad Date: Thu, 20 Jun 2024 11:45:27 -0700 Subject: [PATCH 1/4] Migrated smoke tests for auto detect blob type --- .../zt_newe2e_autodetect_blob_type_test.go | 109 +++++++++++++++++ testSuite/scripts/run.py | 2 - .../scripts/test_autodetect_blob_type.py | 110 ------------------ 3 files changed, 109 insertions(+), 112 deletions(-) create mode 100644 e2etest/zt_newe2e_autodetect_blob_type_test.go delete mode 100644 testSuite/scripts/test_autodetect_blob_type.py diff --git a/e2etest/zt_newe2e_autodetect_blob_type_test.go b/e2etest/zt_newe2e_autodetect_blob_type_test.go new file mode 100644 index 000000000..190e9f19c --- /dev/null +++ b/e2etest/zt_newe2e_autodetect_blob_type_test.go @@ -0,0 +1,109 @@ +package e2etest + +import ( + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-storage-azcopy/v10/common" +) + +func init() { + suiteManager.RegisterSuite(&AutoDetectBlobTypeTestSuite{}) +} + +type AutoDetectBlobTypeTestSuite struct{} + +func (s *AutoDetectBlobTypeTestSuite) Scenario_AutoInferBlobTypeVHD(svm *ScenarioVariationManager) { + fileName := "myVHD.vHd" // awkward capitalization to see if AzCopy catches it. + body := NewRandomObjectContentContainer(svm, 4*common.MegaByte) + + srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). + GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + dstObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}). + GetObject(svm, fileName, common.EEntityType.File()) + + // copy vhd file without specifying page blob. Page blob is inferred for VHD, VHDX, and VMDK + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObj, dstObj}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + BlockSizeMB: pointerTo(4.0), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{ + Body: body, + ObjectProperties: ObjectProperties{ + BlobProperties: BlobProperties{ + Type: pointerTo(blob.BlobTypePageBlob), + }, + }, + }, true) +} + +func (s *AutoDetectBlobTypeTestSuite) Scenario_InferBlobTypeFilePageBlob(svm *ScenarioVariationManager) { + fileName := "testS2SVHD.vhd" + body := NewRandomObjectContentContainer(svm, 4*common.MegaByte) + + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{}) + srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcContainer, dstContainer}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ + Body: body, + ObjectProperties: ObjectProperties{ + BlobProperties: BlobProperties{ + Type: pointerTo(blob.BlobTypePageBlob), + }, + }, + }, true) +} + +func (s *AutoDetectBlobTypeTestSuite) Scenario_DetectBlobTypeBlobBlob(svm *ScenarioVariationManager) { + fileName := "testS2SVHD.vhd" + body := NewRandomObjectContentContainer(svm, 4*common.MegaByte) + + // Upload to Azure Blob Storage as Block Blob and detect as Block Blob. + // AzCopy detects the source blob type. + // This means that in all scenarios EXCEPT Blob -> Blob, .vhd corresponds to page blob. + // However, in Blob -> Blob, we preserve the blob type instead of detecting it. + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) + srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcContainer, dstContainer}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ + Body: body, + ObjectProperties: ObjectProperties{ + BlobProperties: BlobProperties{ + Type: pointerTo(blob.BlobTypeBlockBlob), + }, + }, + }, true) +} diff --git a/testSuite/scripts/run.py b/testSuite/scripts/run.py index 899b8e37e..6184a79ac 100644 --- a/testSuite/scripts/run.py +++ b/testSuite/scripts/run.py @@ -14,7 +14,6 @@ from test_google_cloud_storage_copy import * from test_blobfs_download_SAS import * from test_blobfs_upload_SAS import * -from test_autodetect_blob_type import * from test_file_sync import * from test_file_copy import * from test_google_cloud_storage_copy import * @@ -228,7 +227,6 @@ def main(): BlobFs_Upload_ShareKey_User_Scenarios, BlobFs_Download_SharedKey_User_Scenarios, Service_2_Service_Copy_User_Scenario, - Autodetect_Blob_Type_Scenario, Google_Cloud_Storage_Copy_User_Scenario] suites_list = [] diff --git a/testSuite/scripts/test_autodetect_blob_type.py b/testSuite/scripts/test_autodetect_blob_type.py deleted file mode 100644 index afb6e1338..000000000 --- a/testSuite/scripts/test_autodetect_blob_type.py +++ /dev/null @@ -1,110 +0,0 @@ -import filecmp -import os -import unittest -import utility as util - -class Autodetect_Blob_Type_Scenario(unittest.TestCase): - def setUp(self): - cmd = util.Command("login").add_arguments("--service-principal").add_flags("application-id", os.environ['ACTIVE_DIRECTORY_APPLICATION_ID']).add_flags("tenant-id", os.environ['OAUTH_TENANT_ID']) - cmd.execute_azcopy_copy_command() - - def tearDown(self): - cmd = util.Command("logout") - cmd.execute_azcopy_copy_command() - - # Currently, the only auto-detected blob type is page blob. - # Copy a VHD without specifying page blob and see what it does. - def test_auto_infer_blob_type_vhd(self): - # Upload to blob storage, detecting as a page blob. - file_name = "myVHD.vHd" # awkward capitalization to see if azcopy catches it - file_path = util.create_test_file(file_name, 4 * 1024 * 1024) - - # copy VHD file without specifying page blob. Page blob is inferred for VHD, VHDX, and VMDK - destination_sas = util.get_object_sas(util.test_container_url, file_name) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("log-level", - "info"). \ - add_flags("block-size-mb", 4).execute_azcopy_copy_command() - self.assertTrue(result) - - # execute validator. Validator will ensure it's a page blob as validator now checks blob type before testing. - result = util.Command("testBlob").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("blob-type", "PageBlob").execute_azcopy_verify() - self.assertTrue(result) - - def test_copy_infer_blob_type_from_files_to_page_blob(self): - # Upload to Azure Files and then transfer to Blob to detect as page blob. - file_name = "testS2SVHD.vhd" - containerName = util.get_resource_name("s2sbtautodetect") - - # These run on separate accounts in CI, so even without "dst", it's OK. - # Needed this to run on a single account, though. - dstbase = util.get_object_sas(util.test_s2s_dst_blob_account_url, containerName + "dst") - srcbase = util.get_object_sas(util.test_s2s_src_file_account_url, containerName) - - result = util.Command("create").add_arguments(srcbase).add_flags("serviceType", "File"). \ - add_flags("resourceType", "Bucket").execute_azcopy_create() - self.assertTrue(result) - - result = util.Command("create").add_arguments(dstbase).add_flags("serviceType", "Blob"). \ - add_flags("resourceType", "Bucket").execute_azcopy_create() - self.assertTrue(result) - - src_container_url = util.get_object_sas(srcbase, file_name) - dst_container_url = util.get_object_sas(dstbase, file_name) - - file_path = util.create_test_file(file_name, 4 * 1024 * 1024) - - result = util.Command("copy").add_arguments(file_path).add_arguments(src_container_url). \ - add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - result = util.Command("copy").add_arguments(src_container_url).add_arguments(dst_container_url). \ - add_flags("log-level", "info").execute_azcopy_copy_command() # Blob type in this case will be inferred. - self.assertTrue(result) - - # Verify blob type (should be page blob) - result = util.Command("testBlob").add_arguments(file_path).add_arguments(dst_container_url). \ - add_flags("blob-type", "PageBlob").execute_azcopy_verify() - self.assertTrue(result) - - def test_copy_detect_blob_type_from_blob_to_blob(self): - # Upload to Azure Blob Storage as Block Blob and detect as Block Blob. - # Why is this necessary, you ask? - # Because we detect the source blob type. - # This means that in all scenarios EXCEPT Blob -> Blob, .vhd corresponds to page blob. - # However, in Blob -> Blob, we preserve the blob type instead of detecting it. - # As a result, this behavior should be tested. - file_name = "testS2SVHD.vhd" - containerName = util.get_resource_name("s2sbtautodetect") - - # These run on separate accounts in CI, so even without "dst", it's OK. - # Needed this to run on a single account, though. - dstbase = util.get_object_sas(util.test_s2s_dst_blob_account_url, containerName + "dst") - srcbase = util.get_object_sas(util.test_s2s_src_blob_account_url, containerName) - - result = util.Command("create").add_arguments(srcbase).add_flags("serviceType", "Blob"). \ - add_flags("resourceType", "Bucket").execute_azcopy_create() - self.assertTrue(result) - - result = util.Command("create").add_arguments(dstbase).add_flags("serviceType", "Blob"). \ - add_flags("resourceType", "Bucket").execute_azcopy_create() - self.assertTrue(result) - - src_container_url = util.get_object_sas(srcbase, file_name) - dst_container_url = util.get_object_sas(dstbase, file_name) - - file_path = util.create_test_file(file_name, 4 * 1024 * 1024) - - # Explicitly define BlockBlob as this is local -> blob - result = util.Command("copy").add_arguments(file_path).add_arguments(src_container_url). \ - add_flags("blob-type", "BlockBlob").add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - result = util.Command("copy").add_arguments(src_container_url).add_arguments(dst_container_url). \ - add_flags("log-level", "info").execute_azcopy_copy_command() # Blob type will be detected as block. - self.assertTrue(result) - - # Verify blob type (should be block blob) - result = util.Command("testBlob").add_arguments(file_path).add_arguments(dst_container_url). \ - add_flags("blob-type", "BlockBlob").execute_azcopy_verify() - self.assertTrue(result) From 0fb89b9cb947473330c0dd5d457c73cf134d73a5 Mon Sep 17 00:00:00 2001 From: Gauri Prasad Date: Thu, 20 Jun 2024 13:57:14 -0700 Subject: [PATCH 2/4] Migrated blobfs smoke tests --- common/fe-ste-models.go | 1 + e2etest/zt_newe2e_blobfs_test.go | 244 ++++++++++++++++++ testSuite/scripts/run.py | 8 - testSuite/scripts/test_blobfs_download_SAS.py | 108 -------- .../scripts/test_blobfs_download_oauth.py | 190 -------------- testSuite/scripts/test_blobfs_upload_SAS.py | 60 ----- testSuite/scripts/test_blobfs_upload_oauth.py | 156 ----------- 7 files changed, 245 insertions(+), 522 deletions(-) create mode 100644 e2etest/zt_newe2e_blobfs_test.go delete mode 100644 testSuite/scripts/test_blobfs_download_SAS.py delete mode 100644 testSuite/scripts/test_blobfs_download_oauth.py delete mode 100644 testSuite/scripts/test_blobfs_upload_SAS.py delete mode 100644 testSuite/scripts/test_blobfs_upload_oauth.py diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index d266ab044..262a52b67 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -1112,6 +1112,7 @@ const ( BlockSizeThreshold = 256 * 1024 * 1024 MinParallelChunkCountThreshold = 4 /* minimum number of chunks in parallel for AzCopy to be performant. */ MegaByte = 1024 * 1024 + KiloByte = 1024 ) // This struct represent a single transfer entry with source and destination details diff --git a/e2etest/zt_newe2e_blobfs_test.go b/e2etest/zt_newe2e_blobfs_test.go new file mode 100644 index 000000000..a128edfa4 --- /dev/null +++ b/e2etest/zt_newe2e_blobfs_test.go @@ -0,0 +1,244 @@ +package e2etest + +import ( + "github.com/Azure/azure-storage-azcopy/v10/common" + "strconv" +) + +func init() { + suiteManager.RegisterSuite(&BlobFSTestSuite{}) +} + +type BlobFSTestSuite struct{} + +func (s *BlobFSTestSuite) Scenario_UploadFile(svm *ScenarioVariationManager) { + fileName := "test_1k.txt" + body := NewRandomObjectContentContainer(svm, common.KiloByte) + + srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). + GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + acct := GetAccount(svm, PrimaryHNSAcct) + dstService := acct.GetService(svm, common.ELocation.BlobFS()) + dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{}) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObj, dstContainer.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{})}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ + Body: body, + }, true) +} + +func (s *BlobFSTestSuite) Scenario_UploadFile64MB(svm *ScenarioVariationManager) { + fileName := "test_64mb.txt" + body := NewRandomObjectContentContainer(svm, 64*common.MegaByte) + + srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). + GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + acct := GetAccount(svm, PrimaryHNSAcct) + dstService := acct.GetService(svm, common.ELocation.BlobFS()) + dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{}) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObj, dstContainer.(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{})}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ + Body: body, + }, true) +} + +func (s *BlobFSTestSuite) Scenario_UploadFileUnevenMultiflushOAuth(svm *ScenarioVariationManager) { + fileName := "test_uneven_multiflush_64MB_file.txt" + body := NewRandomObjectContentContainer(svm, 64*common.MegaByte) + + srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). + GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + acct := GetAccount(svm, PrimaryHNSAcct) + dstService := acct.GetService(svm, common.ELocation.BlobFS()) + dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{}) + + // Upload the file using AzCopy @ 1MB blocks, 15 block flushes (5 flushes, 4 15 blocks, 1 4 blocks) + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObj, dstContainer.(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.OAuth(), svm, CreateAzCopyTargetOptions{})}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + BlockSizeMB: pointerTo(1.0), + }, + ADLSFlushThreshold: pointerTo(uint32(15)), + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ + Body: body, + }, true) +} + +func (s *BlobFSTestSuite) Scenario_UploadFileEvenMultiflushOAuth(svm *ScenarioVariationManager) { + fileName := "test_even_multiflush_64MB_file.txt" + body := NewRandomObjectContentContainer(svm, 64*common.MegaByte) + + srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). + GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + acct := GetAccount(svm, PrimaryHNSAcct) + dstService := acct.GetService(svm, common.ELocation.BlobFS()) + dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{}) + + // Upload the file using AzCopy @ 1MB blocks, 16 block flushes (4 16 block flushes) + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObj, dstContainer.(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.OAuth(), svm, CreateAzCopyTargetOptions{})}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + BlockSizeMB: pointerTo(1.0), + }, + ADLSFlushThreshold: pointerTo(uint32(16)), + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ + Body: body, + }, true) +} + +func (s *BlobFSTestSuite) Scenario_Upload100Files(svm *ScenarioVariationManager) { + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}) + acct := GetAccount(svm, PrimaryHNSAcct) + dstService := acct.GetService(svm, common.ELocation.BlobFS()) + dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{}) + + srcObject := srcContainer.GetObject(svm, "dir_100_files", common.EEntityType.Folder()) + + srcObjs := make(ObjectResourceMappingFlat) + for i := range 100 { + name := "dir_100_files/test" + strconv.Itoa(i) + ".txt" + obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: NewRandomObjectContentContainer(svm, SizeFromString("1K"))} + CreateResource[ObjectResourceManager](svm, srcContainer, obj) + srcObjs[name] = obj + } + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObject, dstContainer.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{})}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ContainerResourceManager](svm, dstContainer, ResourceDefinitionContainer{ + Objects: srcObjs, + }, true) +} + +func (s *BlobFSTestSuite) Scenario_DownloadFile(svm *ScenarioVariationManager) { + fileName := "test_1k.txt" + body := NewRandomObjectContentContainer(svm, common.KiloByte) + + dstObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). + GetObject(svm, fileName, common.EEntityType.File()) + + acct := GetAccount(svm, PrimaryHNSAcct) + srcService := acct.GetService(svm, common.ELocation.BlobFS()) + srcContainer := CreateResource[ContainerResourceManager](svm, srcService, ResourceDefinitionContainer{}) + srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObj.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{}), dstObj}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{ + Body: body, + }, true) +} + +func (s *BlobFSTestSuite) Scenario_DownloadFile64MB(svm *ScenarioVariationManager) { + fileName := "test_64mb.txt" + body := NewRandomObjectContentContainer(svm, common.KiloByte) + + dstObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). + GetObject(svm, fileName, common.EEntityType.File()) + + acct := GetAccount(svm, PrimaryHNSAcct) + srcService := acct.GetService(svm, common.ELocation.BlobFS()) + srcContainer := CreateResource[ContainerResourceManager](svm, srcService, ResourceDefinitionContainer{}) + srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObj.(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{}), dstObj}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{ + Body: body, + }, true) +} + +func (s *BlobFSTestSuite) Scenario_Download100Files(svm *ScenarioVariationManager) { + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}) + acct := GetAccount(svm, PrimaryHNSAcct) + srcService := acct.GetService(svm, common.ELocation.BlobFS()) + srcContainer := CreateResource[ContainerResourceManager](svm, srcService, ResourceDefinitionContainer{}) + + srcObject := srcContainer.GetObject(svm, "dir_100_files", common.EEntityType.Folder()) + + srcObjs := make(ObjectResourceMappingFlat) + for i := range 100 { + name := "dir_100_files/test" + strconv.Itoa(i) + ".txt" + obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: NewRandomObjectContentContainer(svm, SizeFromString("1K"))} + CreateResource[ObjectResourceManager](svm, srcContainer, obj) + srcObjs[name] = obj + } + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObject.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{}), dstContainer}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ContainerResourceManager](svm, dstContainer, ResourceDefinitionContainer{ + Objects: srcObjs, + }, true) +} diff --git a/testSuite/scripts/run.py b/testSuite/scripts/run.py index 6184a79ac..967355d13 100644 --- a/testSuite/scripts/run.py +++ b/testSuite/scripts/run.py @@ -5,15 +5,11 @@ from test_file_upload import * from test_azcopy_operations import * from test_blobfs_upload_sharedkey import * -from test_blobfs_upload_oauth import * from test_blobfs_download_sharedkey import * -from test_blobfs_download_oauth import * from test_blob_piping import * from test_blob_sync import * from test_service_to_service_copy import * from test_google_cloud_storage_copy import * -from test_blobfs_download_SAS import * -from test_blobfs_upload_SAS import * from test_file_sync import * from test_file_copy import * from test_google_cloud_storage_copy import * @@ -217,10 +213,6 @@ def main(): Block_Upload_User_Scenarios, Blob_Download_User_Scenario, PageBlob_Upload_User_Scenarios, - BlobFs_Upload_OAuth_User_Scenarios, - BlobFs_Download_OAuth_User_Scenarios, - BlobFs_Download_SAS_User_Scenarios, - BlobFs_Upload_SAS_User_Scenarios, Azcopy_Operation_User_Scenario, FileShare_Download_User_Scenario, FileShare_Upload_User_Scenario, diff --git a/testSuite/scripts/test_blobfs_download_SAS.py b/testSuite/scripts/test_blobfs_download_SAS.py deleted file mode 100644 index 0b396da7a..000000000 --- a/testSuite/scripts/test_blobfs_download_SAS.py +++ /dev/null @@ -1,108 +0,0 @@ -import unittest -import os -import utility as util -import shutil - -class BlobFs_Download_SAS_User_Scenarios(unittest.TestCase): - def setUp(self): - # ensure OAuth is not being used - self.cachedAzCopyClientSecret = os.environ['AZCOPY_SPA_CLIENT_SECRET'] - os.environ['AZCOPY_SPA_CLIENT_SECRET'] = '' - # ensure account key is not being used - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def tearDown(self): - os.environ['AZCOPY_SPA_CLIENT_SECRET'] = self.cachedAzCopyClientSecret - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - - def test_blobfs_sas_download_1Kb_file(self): - # Create file of size 1KB - filename = "test_blobfs_sas_d_1kb_file.txt" - file_path = util.create_test_file(filename, 1024) - # Upload the file using azCopy - result = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_sas_account_url). \ - add_flags("log-level", "Info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded file - file_url = util.get_resource_sas_from_bfs(filename) - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(file_url).execute_azcopy_verify() - self.assertTrue(result) - - # Delete the local file - try: - os.remove(file_path) - except: - self.fail('error deleting the file ' + file_path) - - # Download the file using AzCopy - result = util.Command("copy").add_arguments(file_url).add_arguments(util.test_directory_path). \ - add_flags("log-level", "Info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the downloaded file - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(file_url).execute_azcopy_verify() - self.assertTrue(result) - - def test_blobfs_sas_download_64MB_file(self): - # Create file of size 1KB - filename = "test_blobfs_sas_d_64MB_file.txt" - file_path = util.create_test_file(filename, 64*1024*1024) - # Upload the file using azCopy - result = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_sas_account_url). \ - add_flags("log-level", "Info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded file - file_url = util.get_resource_sas_from_bfs(filename) - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(file_url).execute_azcopy_verify() - self.assertTrue(result) - - # Delete the local file - try: - os.remove(file_path) - except: - self.fail('error deleting the file ' + file_path) - - # Download the file using AzCopy - result = util.Command("copy").add_arguments(file_url).add_arguments(util.test_directory_path). \ - add_flags("log-level", "Info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the downloaded file - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(file_url).execute_azcopy_verify() - self.assertTrue(result) - - def test_blobfs_sas_download_100_1Kb_file(self): - # Create dir with 100 1kb files inside it - dir_name = "dir_blobfs_sas_d_100_1K" - dir_n_file_path = util.create_test_n_files(1024, 100, dir_name) - - # Upload the directory with 100 files inside it - result = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_sas_account_url). \ - add_flags("log-level", "Info").add_flags("recursive", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded directory - dirUrl = util.get_resource_sas_from_bfs(dir_name) - dirUrl_nosas = util.test_bfs_account_url + dir_name - result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # Delete the local files - try: - shutil.rmtree(dir_n_file_path) - except: - self.fail('error deleting the directory ' + dir_n_file_path) - - # Download the directory - result = util.Command("copy").add_arguments(dirUrl).add_arguments(util.test_directory_path). \ - add_flags("log-level", "Info").add_flags("recursive", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the downloaded directory - result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) diff --git a/testSuite/scripts/test_blobfs_download_oauth.py b/testSuite/scripts/test_blobfs_download_oauth.py deleted file mode 100644 index 4c6f2c569..000000000 --- a/testSuite/scripts/test_blobfs_download_oauth.py +++ /dev/null @@ -1,190 +0,0 @@ -import os -import shutil -import sys -import unittest -import utility as util - -class BlobFs_Download_OAuth_User_Scenarios(unittest.TestCase): - def setUp(self): - # ensure account key is not being used - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - os.environ['AZCOPY_AUTO_LOGIN_TYPE'] = 'SPN' - os.environ['AZCOPY_SPA_APPLICATION_ID'] = os.environ['ACTIVE_DIRECTORY_APPLICATION_ID'] - os.environ['AZCOPY_TENANT_ID'] = os.environ['OAUTH_TENANT_ID'] - - def tearDown(self): - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - os.environ['AZCOPY_AUTO_LOGIN_TYPE'] = '' - os.environ['AZCOPY_SPA_APPLICATION_ID'] = '' - os.environ['AZCOPY_TENANT_ID'] = '' - - def util_test_blobfs_download_1Kb_file( - self, - explictFromTo=False): - # create file of size 1KB - filename = "test_blob_d_1kb_file.txt" - file_path = util.create_test_file(filename, 1024) - # upload the file using Azcopy - cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the file uploaded file - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - fileUrl = util.test_bfs_account_url + filename - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - # delete the file locally - try: - os.remove(file_path) - except: - self.fail('error deleting the file ' + file_path) - - # download the file using Azcopy - cmd = util.Command("copy").add_arguments(fileUrl).add_arguments(util.test_directory_path). \ - add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "BlobFSLocal" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # validate the downloaded file - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def util_test_blobfs_download_64MB_file( - self, - explictFromTo=False): - # create test file of size 64MB - filename = "test_blob_d_64MB_file.txt" - file_path = util.create_test_file(filename, 64*1024*1024) - # Upload the file using Azcopy - cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the file uploaded - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - fileUrl = util.test_bfs_account_url + filename - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - # delete the file locally - try: - os.remove(file_path) - except: - self.fail('error deleting the file ' + file_path) - - # download the file using azcopy - cmd = util.Command("copy").add_arguments(fileUrl).add_arguments(util.test_directory_path). \ - add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "BlobFSLocal" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # validate the downloaded file - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def util_test_blobfs_download_100_1Kb_file( - self, - explictFromTo=False): - # create dir with 100 1KB files inside it - dir_name = "dir_blobfs_d_100_1K" - dir_n_file_path = util.create_test_n_files(1024, 100, dir_name) - - # Upload the directory with 100 files inside it - cmd = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("log-level", "Info").add_flags("recursive","true") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded directory - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - dirUrl = util.test_bfs_account_url + dir_name - result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - # delete the local directory created - try: - shutil.rmtree(dir_n_file_path) - except: - self.fail('error deleting the directory ' + dir_n_file_path) - - # download the directory - cmd = util.Command("copy").add_arguments(dirUrl).add_arguments(util.test_directory_path).\ - add_flags("log-level", "Info").add_flags("recursive", "true") - util.process_oauth_command( - cmd, - "BlobFSLocal" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # validate the downloaded directory - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl).\ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def test_blobfs_download_200_1Kb_file(self): - # create dir with 100 1KB files inside it - dir_name = "dir_blobfs_200_1K" - dir_n_file_path = util.create_test_n_files(1024, 200, dir_name) - - # Upload the directory with 2000 files inside it - result = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("log-level", "Info").add_flags("recursive","true").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded directory - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - dirUrl = util.test_bfs_account_url + dir_name - result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - # Using shared key only when oauth token is not preset - def test_blobfs_download_1Kb_file_with_oauth(self): - self.util_test_blobfs_download_1Kb_file() - - # Using shared key only when oauth token is not preset - @unittest.skip("single file authentication for oauth covered by 1kb case") - def test_blobfs_download_64MB_file_with_oauth(self): - self.util_test_blobfs_download_64MB_file() - - # Using shared key only when oauth token is not preset - def test_blobfs_download_100_1Kb_file_with_oauth(self): - self.util_test_blobfs_download_100_1Kb_file() \ No newline at end of file diff --git a/testSuite/scripts/test_blobfs_upload_SAS.py b/testSuite/scripts/test_blobfs_upload_SAS.py deleted file mode 100644 index 15fbeac6f..000000000 --- a/testSuite/scripts/test_blobfs_upload_SAS.py +++ /dev/null @@ -1,60 +0,0 @@ -import unittest -import os -import utility as util - -class BlobFs_Upload_SAS_User_Scenarios(unittest.TestCase): - def setUp(self): - # ensure OAuth is not being used - self.cachedAzCopyClientSecret = os.environ['AZCOPY_SPA_CLIENT_SECRET'] - os.environ['AZCOPY_SPA_CLIENT_SECRET'] = '' - # ensure account key is not being used - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def tearDown(self): - os.environ['AZCOPY_SPA_CLIENT_SECRET'] = self.cachedAzCopyClientSecret - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - - def test_blobfs_sas_upload_1Kb_file(self): - # Create file of size 1KB - filename = "test_blobfs_sas_u_1kb_file.txt" - file_path = util.create_test_file(filename, 1024) - # Upload the file using azCopy - result = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_sas_account_url). \ - add_flags("log-level", "Info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded file - file_url = util.get_resource_sas_from_bfs(filename) - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(file_url).execute_azcopy_verify() - self.assertTrue(result) - - def test_blobfs_sas_upload_64MB_file(self): - # Create file of size 1KB - filename = "test_blobfs_sas_u_64MB_file.txt" - file_path = util.create_test_file(filename, 64 * 1024 * 1024) - # Upload the file using azCopy - result = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_sas_account_url). \ - add_flags("log-level", "Info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded file - file_url = util.get_resource_sas_from_bfs(filename) - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(file_url).execute_azcopy_verify() - self.assertTrue(result) - - def test_blobfs_sas_upload_100_1kb_file(self): - # Create dir with 100 1kb files inside it - dir_name = "dir_blobfs_sas_u_100_1K" - dir_n_file_path = util.create_test_n_files(1024, 100, dir_name) - - # Upload the directory with 100 files inside it - result = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_sas_account_url). \ - add_flags("log-level", "Info").add_flags("recursive", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded directory - dirUrl = util.get_resource_sas_from_bfs(dir_name) - result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) diff --git a/testSuite/scripts/test_blobfs_upload_oauth.py b/testSuite/scripts/test_blobfs_upload_oauth.py deleted file mode 100644 index 8663a8b41..000000000 --- a/testSuite/scripts/test_blobfs_upload_oauth.py +++ /dev/null @@ -1,156 +0,0 @@ -import json -import os -import shutil -from collections import namedtuple -from stat import * -import sys -import utility as util -import unittest - -class BlobFs_Upload_OAuth_User_Scenarios(unittest.TestCase): - def setUp(self): - # ensure account key is not being used - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - os.environ['AZCOPY_AUTO_LOGIN_TYPE'] = 'SPN' - os.environ['AZCOPY_SPA_APPLICATION_ID'] = os.environ['ACTIVE_DIRECTORY_APPLICATION_ID'] - os.environ['AZCOPY_TENANT_ID'] = os.environ['OAUTH_TENANT_ID'] - - def tearDown(self): - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - os.environ['AZCOPY_AUTO_LOGIN_TYPE'] = '' - os.environ['AZCOPY_SPA_APPLICATION_ID'] = '' - os.environ['AZCOPY_TENANT_ID'] = '' - - def util_test_blobfs_upload_1Kb_file( - self, - explictFromTo=False): - # create file of size 1KB - filename = "test_blob_1kb_file.txt" - file_path = util.create_test_file(filename, 1024) - # upload the file using Azcopy - cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the file uploaded file - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - fileUrl = util.test_bfs_account_url + filename - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def util_test_blobfs_upload_64MB_file( - self, - explictFromTo=False): - # create test file of size 64MB - filename = "test_blob_64MB_file.txt" - file_path = util.create_test_file(filename, 64*1024*1024) - # Upload the file using Azcopy - cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the file uploaded - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - fileUrl = util.test_bfs_account_url + filename - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def util_test_blobfs_upload_uneven_multiflush_file( - self, - explicitFromTo=False): - # create test file of size 64MB - filename = "test_uneven_multiflush_64MB_file.txt" - file_path = util.create_test_file(filename, 64*1024*1024) - # Upload the file using AzCopy @ 1MB blocks, 15 block flushes (5 flushes, 4 15 blocks, 1 4 blocks) - cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("block-size-mb", "1").add_flags("flush-threshold", "15").add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explicitFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the file uploaded - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - fileUrl = util.test_bfs_account_url + filename - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def util_test_blobfs_upload_even_multiflush_file( - self, - explicitFromTo=False): - # create test file of size 64MB - filename = "test_even_multiflush_64MB_file.txt" - file_path = util.create_test_file(filename, 64 * 1024 * 1024) - # Upload the file using AzCopy @ 1MB blocks, 16 block flushes (4 16 block flushes) - cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("block-size-mb", "1").add_flags("flush-threshold", "16").add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explicitFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the file uploaded - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - fileUrl = util.test_bfs_account_url + filename - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def util_test_blobfs_upload_100_1Kb_file( - self, - explictFromTo=False): - # create dir with 100 1KB files inside it - dir_name = "dir_blobfs_100_1K" - dir_n_file_path = util.create_test_n_files(1024, 100, dir_name) - - # Upload the directory with 100 files inside it - cmd = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("log-level", "Info").add_flags("recursive","true") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded directory - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - dirUrl = util.test_bfs_account_url + dir_name - result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def test_blobfs_upload_1Kb_file_with_oauth(self): - self.util_test_blobfs_upload_1Kb_file() - - @unittest.skip("single file authentication for oauth covered by 1kb case") - def test_blobfs_upload_64MB_file_with_oauth(self): - self.util_test_blobfs_upload_64MB_file() - - def test_blobfs_upload_100_1Kb_file_with_oauth(self): - self.util_test_blobfs_upload_100_1Kb_file() - - def test_blobfs_upload_uneven_multiflush_with_oauth(self): - self.util_test_blobfs_upload_uneven_multiflush_file() - - def test_blobfs_upload_even_multiflush_with_oauth(self): - self.util_test_blobfs_upload_even_multiflush_file() From 08e6f7f053f3f388fd291cea39096fb6a3ddab57 Mon Sep 17 00:00:00 2001 From: Gauri Prasad Date: Tue, 16 Jul 2024 13:55:05 -0700 Subject: [PATCH 3/4] combine tests --- .../zt_newe2e_autodetect_blob_type_test.go | 70 +------------------ 1 file changed, 3 insertions(+), 67 deletions(-) diff --git a/e2etest/zt_newe2e_autodetect_blob_type_test.go b/e2etest/zt_newe2e_autodetect_blob_type_test.go index 190e9f19c..1762209e1 100644 --- a/e2etest/zt_newe2e_autodetect_blob_type_test.go +++ b/e2etest/zt_newe2e_autodetect_blob_type_test.go @@ -11,11 +11,11 @@ func init() { type AutoDetectBlobTypeTestSuite struct{} -func (s *AutoDetectBlobTypeTestSuite) Scenario_AutoInferBlobTypeVHD(svm *ScenarioVariationManager) { +func (s *AutoDetectBlobTypeTestSuite) Scenario_AutoInferDetectBlobTypeVHD(svm *ScenarioVariationManager) { fileName := "myVHD.vHd" // awkward capitalization to see if AzCopy catches it. body := NewRandomObjectContentContainer(svm, 4*common.MegaByte) - srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). + srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, ResolveVariation(svm, []common.Location{common.ELocation.Local(), common.ELocation.File(), common.ELocation.Blob()})), ResourceDefinitionContainer{}). GetObject(svm, fileName, common.EEntityType.File()) srcObj.Create(svm, body, ObjectProperties{}) @@ -38,71 +38,7 @@ func (s *AutoDetectBlobTypeTestSuite) Scenario_AutoInferBlobTypeVHD(svm *Scenari Body: body, ObjectProperties: ObjectProperties{ BlobProperties: BlobProperties{ - Type: pointerTo(blob.BlobTypePageBlob), - }, - }, - }, true) -} - -func (s *AutoDetectBlobTypeTestSuite) Scenario_InferBlobTypeFilePageBlob(svm *ScenarioVariationManager) { - fileName := "testS2SVHD.vhd" - body := NewRandomObjectContentContainer(svm, 4*common.MegaByte) - - srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{}) - srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) - srcObj.Create(svm, body, ObjectProperties{}) - - dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) - - RunAzCopy(svm, AzCopyCommand{ - Verb: AzCopyVerbCopy, - Targets: []ResourceManager{srcContainer, dstContainer}, - Flags: CopyFlags{ - CopySyncCommonFlags: CopySyncCommonFlags{ - Recursive: pointerTo(true), - }, - }, - }) - - ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ - Body: body, - ObjectProperties: ObjectProperties{ - BlobProperties: BlobProperties{ - Type: pointerTo(blob.BlobTypePageBlob), - }, - }, - }, true) -} - -func (s *AutoDetectBlobTypeTestSuite) Scenario_DetectBlobTypeBlobBlob(svm *ScenarioVariationManager) { - fileName := "testS2SVHD.vhd" - body := NewRandomObjectContentContainer(svm, 4*common.MegaByte) - - // Upload to Azure Blob Storage as Block Blob and detect as Block Blob. - // AzCopy detects the source blob type. - // This means that in all scenarios EXCEPT Blob -> Blob, .vhd corresponds to page blob. - // However, in Blob -> Blob, we preserve the blob type instead of detecting it. - srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) - srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) - srcObj.Create(svm, body, ObjectProperties{}) - - dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) - - RunAzCopy(svm, AzCopyCommand{ - Verb: AzCopyVerbCopy, - Targets: []ResourceManager{srcContainer, dstContainer}, - Flags: CopyFlags{ - CopySyncCommonFlags: CopySyncCommonFlags{ - Recursive: pointerTo(true), - }, - }, - }) - - ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ - Body: body, - ObjectProperties: ObjectProperties{ - BlobProperties: BlobProperties{ - Type: pointerTo(blob.BlobTypeBlockBlob), + Type: common.Iff(srcObj.Location() == common.ELocation.Blob(), pointerTo(blob.BlobTypeBlockBlob), pointerTo(blob.BlobTypePageBlob)), }, }, }, true) From aeb66408be45eaaf808d76f70701e61a3c3c1aca Mon Sep 17 00:00:00 2001 From: Gauri Prasad Date: Tue, 16 Jul 2024 14:34:09 -0700 Subject: [PATCH 4/4] Addressed comments --- e2etest/zt_newe2e_blobfs_test.go | 101 +++---------------------------- 1 file changed, 10 insertions(+), 91 deletions(-) diff --git a/e2etest/zt_newe2e_blobfs_test.go b/e2etest/zt_newe2e_blobfs_test.go index a128edfa4..dfba6d850 100644 --- a/e2etest/zt_newe2e_blobfs_test.go +++ b/e2etest/zt_newe2e_blobfs_test.go @@ -12,8 +12,9 @@ func init() { type BlobFSTestSuite struct{} func (s *BlobFSTestSuite) Scenario_UploadFile(svm *ScenarioVariationManager) { - fileName := "test_1k.txt" - body := NewRandomObjectContentContainer(svm, common.KiloByte) + fileName := "test.txt" + size := ResolveVariation(svm, []int64{common.KiloByte, 64 * common.MegaByte}) + body := NewRandomObjectContentContainer(svm, size) srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). GetObject(svm, fileName, common.EEntityType.File()) @@ -38,8 +39,8 @@ func (s *BlobFSTestSuite) Scenario_UploadFile(svm *ScenarioVariationManager) { }, true) } -func (s *BlobFSTestSuite) Scenario_UploadFile64MB(svm *ScenarioVariationManager) { - fileName := "test_64mb.txt" +func (s *BlobFSTestSuite) Scenario_UploadFileMultiflushOAuth(svm *ScenarioVariationManager) { + fileName := "test_multiflush_64MB_file.txt" body := NewRandomObjectContentContainer(svm, 64*common.MegaByte) srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). @@ -50,32 +51,7 @@ func (s *BlobFSTestSuite) Scenario_UploadFile64MB(svm *ScenarioVariationManager) dstService := acct.GetService(svm, common.ELocation.BlobFS()) dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{}) - RunAzCopy(svm, AzCopyCommand{ - Verb: AzCopyVerbCopy, - Targets: []ResourceManager{srcObj, dstContainer.(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{})}, - Flags: CopyFlags{ - CopySyncCommonFlags: CopySyncCommonFlags{ - Recursive: pointerTo(true), - }, - }, - }) - - ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ - Body: body, - }, true) -} - -func (s *BlobFSTestSuite) Scenario_UploadFileUnevenMultiflushOAuth(svm *ScenarioVariationManager) { - fileName := "test_uneven_multiflush_64MB_file.txt" - body := NewRandomObjectContentContainer(svm, 64*common.MegaByte) - - srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). - GetObject(svm, fileName, common.EEntityType.File()) - srcObj.Create(svm, body, ObjectProperties{}) - - acct := GetAccount(svm, PrimaryHNSAcct) - dstService := acct.GetService(svm, common.ELocation.BlobFS()) - dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{}) + flushThreshold := ResolveVariation(svm, []uint32{15, 16}) // uneven, even // Upload the file using AzCopy @ 1MB blocks, 15 block flushes (5 flushes, 4 15 blocks, 1 4 blocks) RunAzCopy(svm, AzCopyCommand{ @@ -86,37 +62,7 @@ func (s *BlobFSTestSuite) Scenario_UploadFileUnevenMultiflushOAuth(svm *Scenario Recursive: pointerTo(true), BlockSizeMB: pointerTo(1.0), }, - ADLSFlushThreshold: pointerTo(uint32(15)), - }, - }) - - ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ - Body: body, - }, true) -} - -func (s *BlobFSTestSuite) Scenario_UploadFileEvenMultiflushOAuth(svm *ScenarioVariationManager) { - fileName := "test_even_multiflush_64MB_file.txt" - body := NewRandomObjectContentContainer(svm, 64*common.MegaByte) - - srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). - GetObject(svm, fileName, common.EEntityType.File()) - srcObj.Create(svm, body, ObjectProperties{}) - - acct := GetAccount(svm, PrimaryHNSAcct) - dstService := acct.GetService(svm, common.ELocation.BlobFS()) - dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{}) - - // Upload the file using AzCopy @ 1MB blocks, 16 block flushes (4 16 block flushes) - RunAzCopy(svm, AzCopyCommand{ - Verb: AzCopyVerbCopy, - Targets: []ResourceManager{srcObj, dstContainer.(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.OAuth(), svm, CreateAzCopyTargetOptions{})}, - Flags: CopyFlags{ - CopySyncCommonFlags: CopySyncCommonFlags{ - Recursive: pointerTo(true), - BlockSizeMB: pointerTo(1.0), - }, - ADLSFlushThreshold: pointerTo(uint32(16)), + ADLSFlushThreshold: pointerTo(flushThreshold), }, }) @@ -157,8 +103,9 @@ func (s *BlobFSTestSuite) Scenario_Upload100Files(svm *ScenarioVariationManager) } func (s *BlobFSTestSuite) Scenario_DownloadFile(svm *ScenarioVariationManager) { - fileName := "test_1k.txt" - body := NewRandomObjectContentContainer(svm, common.KiloByte) + fileName := "test.txt" + size := ResolveVariation(svm, []int64{common.KiloByte, 64 * common.MegaByte}) + body := NewRandomObjectContentContainer(svm, size) dstObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). GetObject(svm, fileName, common.EEntityType.File()) @@ -184,34 +131,6 @@ func (s *BlobFSTestSuite) Scenario_DownloadFile(svm *ScenarioVariationManager) { }, true) } -func (s *BlobFSTestSuite) Scenario_DownloadFile64MB(svm *ScenarioVariationManager) { - fileName := "test_64mb.txt" - body := NewRandomObjectContentContainer(svm, common.KiloByte) - - dstObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). - GetObject(svm, fileName, common.EEntityType.File()) - - acct := GetAccount(svm, PrimaryHNSAcct) - srcService := acct.GetService(svm, common.ELocation.BlobFS()) - srcContainer := CreateResource[ContainerResourceManager](svm, srcService, ResourceDefinitionContainer{}) - srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) - srcObj.Create(svm, body, ObjectProperties{}) - - RunAzCopy(svm, AzCopyCommand{ - Verb: AzCopyVerbCopy, - Targets: []ResourceManager{srcObj.(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{}), dstObj}, - Flags: CopyFlags{ - CopySyncCommonFlags: CopySyncCommonFlags{ - Recursive: pointerTo(true), - }, - }, - }) - - ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{ - Body: body, - }, true) -} - func (s *BlobFSTestSuite) Scenario_Download100Files(svm *ScenarioVariationManager) { dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}) acct := GetAccount(svm, PrimaryHNSAcct)