diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index d266ab044..262a52b67 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -1112,6 +1112,7 @@ const ( BlockSizeThreshold = 256 * 1024 * 1024 MinParallelChunkCountThreshold = 4 /* minimum number of chunks in parallel for AzCopy to be performant. */ MegaByte = 1024 * 1024 + KiloByte = 1024 ) // This struct represent a single transfer entry with source and destination details diff --git a/e2etest/zt_newe2e_blobfs_test.go b/e2etest/zt_newe2e_blobfs_test.go new file mode 100644 index 000000000..dfba6d850 --- /dev/null +++ b/e2etest/zt_newe2e_blobfs_test.go @@ -0,0 +1,163 @@ +package e2etest + +import ( + "github.com/Azure/azure-storage-azcopy/v10/common" + "strconv" +) + +func init() { + suiteManager.RegisterSuite(&BlobFSTestSuite{}) +} + +type BlobFSTestSuite struct{} + +func (s *BlobFSTestSuite) Scenario_UploadFile(svm *ScenarioVariationManager) { + fileName := "test.txt" + size := ResolveVariation(svm, []int64{common.KiloByte, 64 * common.MegaByte}) + body := NewRandomObjectContentContainer(svm, size) + + srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). + GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + acct := GetAccount(svm, PrimaryHNSAcct) + dstService := acct.GetService(svm, common.ELocation.BlobFS()) + dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{}) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObj, dstContainer.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{})}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ + Body: body, + }, true) +} + +func (s *BlobFSTestSuite) Scenario_UploadFileMultiflushOAuth(svm *ScenarioVariationManager) { + fileName := "test_multiflush_64MB_file.txt" + body := NewRandomObjectContentContainer(svm, 64*common.MegaByte) + + srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). + GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + acct := GetAccount(svm, PrimaryHNSAcct) + dstService := acct.GetService(svm, common.ELocation.BlobFS()) + dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{}) + + flushThreshold := ResolveVariation(svm, []uint32{15, 16}) // uneven, even + + // Upload the file using AzCopy @ 1MB blocks, 15 block flushes (5 flushes, 4 15 blocks, 1 4 blocks) + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObj, dstContainer.(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.OAuth(), svm, CreateAzCopyTargetOptions{})}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + BlockSizeMB: pointerTo(1.0), + }, + ADLSFlushThreshold: pointerTo(flushThreshold), + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ + Body: body, + }, true) +} + +func (s *BlobFSTestSuite) Scenario_Upload100Files(svm *ScenarioVariationManager) { + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}) + acct := GetAccount(svm, PrimaryHNSAcct) + dstService := acct.GetService(svm, common.ELocation.BlobFS()) + dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{}) + + srcObject := srcContainer.GetObject(svm, "dir_100_files", common.EEntityType.Folder()) + + srcObjs := make(ObjectResourceMappingFlat) + for i := range 100 { + name := "dir_100_files/test" + strconv.Itoa(i) + ".txt" + obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: NewRandomObjectContentContainer(svm, SizeFromString("1K"))} + CreateResource[ObjectResourceManager](svm, srcContainer, obj) + srcObjs[name] = obj + } + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObject, dstContainer.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{})}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ContainerResourceManager](svm, dstContainer, ResourceDefinitionContainer{ + Objects: srcObjs, + }, true) +} + +func (s *BlobFSTestSuite) Scenario_DownloadFile(svm *ScenarioVariationManager) { + fileName := "test.txt" + size := ResolveVariation(svm, []int64{common.KiloByte, 64 * common.MegaByte}) + body := NewRandomObjectContentContainer(svm, size) + + dstObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). + GetObject(svm, fileName, common.EEntityType.File()) + + acct := GetAccount(svm, PrimaryHNSAcct) + srcService := acct.GetService(svm, common.ELocation.BlobFS()) + srcContainer := CreateResource[ContainerResourceManager](svm, srcService, ResourceDefinitionContainer{}) + srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObj.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{}), dstObj}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{ + Body: body, + }, true) +} + +func (s *BlobFSTestSuite) Scenario_Download100Files(svm *ScenarioVariationManager) { + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}) + acct := GetAccount(svm, PrimaryHNSAcct) + srcService := acct.GetService(svm, common.ELocation.BlobFS()) + srcContainer := CreateResource[ContainerResourceManager](svm, srcService, ResourceDefinitionContainer{}) + + srcObject := srcContainer.GetObject(svm, "dir_100_files", common.EEntityType.Folder()) + + srcObjs := make(ObjectResourceMappingFlat) + for i := range 100 { + name := "dir_100_files/test" + strconv.Itoa(i) + ".txt" + obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: NewRandomObjectContentContainer(svm, SizeFromString("1K"))} + CreateResource[ObjectResourceManager](svm, srcContainer, obj) + srcObjs[name] = obj + } + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObject.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{}), dstContainer}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ContainerResourceManager](svm, dstContainer, ResourceDefinitionContainer{ + Objects: srcObjs, + }, true) +} diff --git a/testSuite/scripts/run.py b/testSuite/scripts/run.py index 6184a79ac..967355d13 100644 --- a/testSuite/scripts/run.py +++ b/testSuite/scripts/run.py @@ -5,15 +5,11 @@ from test_file_upload import * from test_azcopy_operations import * from test_blobfs_upload_sharedkey import * -from test_blobfs_upload_oauth import * from test_blobfs_download_sharedkey import * -from test_blobfs_download_oauth import * from test_blob_piping import * from test_blob_sync import * from test_service_to_service_copy import * from test_google_cloud_storage_copy import * -from test_blobfs_download_SAS import * -from test_blobfs_upload_SAS import * from test_file_sync import * from test_file_copy import * from test_google_cloud_storage_copy import * @@ -217,10 +213,6 @@ def main(): Block_Upload_User_Scenarios, Blob_Download_User_Scenario, PageBlob_Upload_User_Scenarios, - BlobFs_Upload_OAuth_User_Scenarios, - BlobFs_Download_OAuth_User_Scenarios, - BlobFs_Download_SAS_User_Scenarios, - BlobFs_Upload_SAS_User_Scenarios, Azcopy_Operation_User_Scenario, FileShare_Download_User_Scenario, FileShare_Upload_User_Scenario, diff --git a/testSuite/scripts/test_blobfs_download_SAS.py b/testSuite/scripts/test_blobfs_download_SAS.py deleted file mode 100644 index 0b396da7a..000000000 --- a/testSuite/scripts/test_blobfs_download_SAS.py +++ /dev/null @@ -1,108 +0,0 @@ -import unittest -import os -import utility as util -import shutil - -class BlobFs_Download_SAS_User_Scenarios(unittest.TestCase): - def setUp(self): - # ensure OAuth is not being used - self.cachedAzCopyClientSecret = os.environ['AZCOPY_SPA_CLIENT_SECRET'] - os.environ['AZCOPY_SPA_CLIENT_SECRET'] = '' - # ensure account key is not being used - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def tearDown(self): - os.environ['AZCOPY_SPA_CLIENT_SECRET'] = self.cachedAzCopyClientSecret - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - - def test_blobfs_sas_download_1Kb_file(self): - # Create file of size 1KB - filename = "test_blobfs_sas_d_1kb_file.txt" - file_path = util.create_test_file(filename, 1024) - # Upload the file using azCopy - result = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_sas_account_url). \ - add_flags("log-level", "Info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded file - file_url = util.get_resource_sas_from_bfs(filename) - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(file_url).execute_azcopy_verify() - self.assertTrue(result) - - # Delete the local file - try: - os.remove(file_path) - except: - self.fail('error deleting the file ' + file_path) - - # Download the file using AzCopy - result = util.Command("copy").add_arguments(file_url).add_arguments(util.test_directory_path). \ - add_flags("log-level", "Info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the downloaded file - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(file_url).execute_azcopy_verify() - self.assertTrue(result) - - def test_blobfs_sas_download_64MB_file(self): - # Create file of size 1KB - filename = "test_blobfs_sas_d_64MB_file.txt" - file_path = util.create_test_file(filename, 64*1024*1024) - # Upload the file using azCopy - result = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_sas_account_url). \ - add_flags("log-level", "Info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded file - file_url = util.get_resource_sas_from_bfs(filename) - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(file_url).execute_azcopy_verify() - self.assertTrue(result) - - # Delete the local file - try: - os.remove(file_path) - except: - self.fail('error deleting the file ' + file_path) - - # Download the file using AzCopy - result = util.Command("copy").add_arguments(file_url).add_arguments(util.test_directory_path). \ - add_flags("log-level", "Info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the downloaded file - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(file_url).execute_azcopy_verify() - self.assertTrue(result) - - def test_blobfs_sas_download_100_1Kb_file(self): - # Create dir with 100 1kb files inside it - dir_name = "dir_blobfs_sas_d_100_1K" - dir_n_file_path = util.create_test_n_files(1024, 100, dir_name) - - # Upload the directory with 100 files inside it - result = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_sas_account_url). \ - add_flags("log-level", "Info").add_flags("recursive", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded directory - dirUrl = util.get_resource_sas_from_bfs(dir_name) - dirUrl_nosas = util.test_bfs_account_url + dir_name - result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # Delete the local files - try: - shutil.rmtree(dir_n_file_path) - except: - self.fail('error deleting the directory ' + dir_n_file_path) - - # Download the directory - result = util.Command("copy").add_arguments(dirUrl).add_arguments(util.test_directory_path). \ - add_flags("log-level", "Info").add_flags("recursive", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the downloaded directory - result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) diff --git a/testSuite/scripts/test_blobfs_download_oauth.py b/testSuite/scripts/test_blobfs_download_oauth.py deleted file mode 100644 index 4c6f2c569..000000000 --- a/testSuite/scripts/test_blobfs_download_oauth.py +++ /dev/null @@ -1,190 +0,0 @@ -import os -import shutil -import sys -import unittest -import utility as util - -class BlobFs_Download_OAuth_User_Scenarios(unittest.TestCase): - def setUp(self): - # ensure account key is not being used - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - os.environ['AZCOPY_AUTO_LOGIN_TYPE'] = 'SPN' - os.environ['AZCOPY_SPA_APPLICATION_ID'] = os.environ['ACTIVE_DIRECTORY_APPLICATION_ID'] - os.environ['AZCOPY_TENANT_ID'] = os.environ['OAUTH_TENANT_ID'] - - def tearDown(self): - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - os.environ['AZCOPY_AUTO_LOGIN_TYPE'] = '' - os.environ['AZCOPY_SPA_APPLICATION_ID'] = '' - os.environ['AZCOPY_TENANT_ID'] = '' - - def util_test_blobfs_download_1Kb_file( - self, - explictFromTo=False): - # create file of size 1KB - filename = "test_blob_d_1kb_file.txt" - file_path = util.create_test_file(filename, 1024) - # upload the file using Azcopy - cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the file uploaded file - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - fileUrl = util.test_bfs_account_url + filename - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - # delete the file locally - try: - os.remove(file_path) - except: - self.fail('error deleting the file ' + file_path) - - # download the file using Azcopy - cmd = util.Command("copy").add_arguments(fileUrl).add_arguments(util.test_directory_path). \ - add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "BlobFSLocal" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # validate the downloaded file - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def util_test_blobfs_download_64MB_file( - self, - explictFromTo=False): - # create test file of size 64MB - filename = "test_blob_d_64MB_file.txt" - file_path = util.create_test_file(filename, 64*1024*1024) - # Upload the file using Azcopy - cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the file uploaded - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - fileUrl = util.test_bfs_account_url + filename - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - # delete the file locally - try: - os.remove(file_path) - except: - self.fail('error deleting the file ' + file_path) - - # download the file using azcopy - cmd = util.Command("copy").add_arguments(fileUrl).add_arguments(util.test_directory_path). \ - add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "BlobFSLocal" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # validate the downloaded file - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def util_test_blobfs_download_100_1Kb_file( - self, - explictFromTo=False): - # create dir with 100 1KB files inside it - dir_name = "dir_blobfs_d_100_1K" - dir_n_file_path = util.create_test_n_files(1024, 100, dir_name) - - # Upload the directory with 100 files inside it - cmd = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("log-level", "Info").add_flags("recursive","true") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded directory - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - dirUrl = util.test_bfs_account_url + dir_name - result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - # delete the local directory created - try: - shutil.rmtree(dir_n_file_path) - except: - self.fail('error deleting the directory ' + dir_n_file_path) - - # download the directory - cmd = util.Command("copy").add_arguments(dirUrl).add_arguments(util.test_directory_path).\ - add_flags("log-level", "Info").add_flags("recursive", "true") - util.process_oauth_command( - cmd, - "BlobFSLocal" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # validate the downloaded directory - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl).\ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def test_blobfs_download_200_1Kb_file(self): - # create dir with 100 1KB files inside it - dir_name = "dir_blobfs_200_1K" - dir_n_file_path = util.create_test_n_files(1024, 200, dir_name) - - # Upload the directory with 2000 files inside it - result = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("log-level", "Info").add_flags("recursive","true").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded directory - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - dirUrl = util.test_bfs_account_url + dir_name - result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - # Using shared key only when oauth token is not preset - def test_blobfs_download_1Kb_file_with_oauth(self): - self.util_test_blobfs_download_1Kb_file() - - # Using shared key only when oauth token is not preset - @unittest.skip("single file authentication for oauth covered by 1kb case") - def test_blobfs_download_64MB_file_with_oauth(self): - self.util_test_blobfs_download_64MB_file() - - # Using shared key only when oauth token is not preset - def test_blobfs_download_100_1Kb_file_with_oauth(self): - self.util_test_blobfs_download_100_1Kb_file() \ No newline at end of file diff --git a/testSuite/scripts/test_blobfs_upload_SAS.py b/testSuite/scripts/test_blobfs_upload_SAS.py deleted file mode 100644 index 15fbeac6f..000000000 --- a/testSuite/scripts/test_blobfs_upload_SAS.py +++ /dev/null @@ -1,60 +0,0 @@ -import unittest -import os -import utility as util - -class BlobFs_Upload_SAS_User_Scenarios(unittest.TestCase): - def setUp(self): - # ensure OAuth is not being used - self.cachedAzCopyClientSecret = os.environ['AZCOPY_SPA_CLIENT_SECRET'] - os.environ['AZCOPY_SPA_CLIENT_SECRET'] = '' - # ensure account key is not being used - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def tearDown(self): - os.environ['AZCOPY_SPA_CLIENT_SECRET'] = self.cachedAzCopyClientSecret - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - - def test_blobfs_sas_upload_1Kb_file(self): - # Create file of size 1KB - filename = "test_blobfs_sas_u_1kb_file.txt" - file_path = util.create_test_file(filename, 1024) - # Upload the file using azCopy - result = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_sas_account_url). \ - add_flags("log-level", "Info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded file - file_url = util.get_resource_sas_from_bfs(filename) - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(file_url).execute_azcopy_verify() - self.assertTrue(result) - - def test_blobfs_sas_upload_64MB_file(self): - # Create file of size 1KB - filename = "test_blobfs_sas_u_64MB_file.txt" - file_path = util.create_test_file(filename, 64 * 1024 * 1024) - # Upload the file using azCopy - result = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_sas_account_url). \ - add_flags("log-level", "Info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded file - file_url = util.get_resource_sas_from_bfs(filename) - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(file_url).execute_azcopy_verify() - self.assertTrue(result) - - def test_blobfs_sas_upload_100_1kb_file(self): - # Create dir with 100 1kb files inside it - dir_name = "dir_blobfs_sas_u_100_1K" - dir_n_file_path = util.create_test_n_files(1024, 100, dir_name) - - # Upload the directory with 100 files inside it - result = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_sas_account_url). \ - add_flags("log-level", "Info").add_flags("recursive", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded directory - dirUrl = util.get_resource_sas_from_bfs(dir_name) - result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) diff --git a/testSuite/scripts/test_blobfs_upload_oauth.py b/testSuite/scripts/test_blobfs_upload_oauth.py deleted file mode 100644 index 8663a8b41..000000000 --- a/testSuite/scripts/test_blobfs_upload_oauth.py +++ /dev/null @@ -1,156 +0,0 @@ -import json -import os -import shutil -from collections import namedtuple -from stat import * -import sys -import utility as util -import unittest - -class BlobFs_Upload_OAuth_User_Scenarios(unittest.TestCase): - def setUp(self): - # ensure account key is not being used - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - os.environ['AZCOPY_AUTO_LOGIN_TYPE'] = 'SPN' - os.environ['AZCOPY_SPA_APPLICATION_ID'] = os.environ['ACTIVE_DIRECTORY_APPLICATION_ID'] - os.environ['AZCOPY_TENANT_ID'] = os.environ['OAUTH_TENANT_ID'] - - def tearDown(self): - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - os.environ['AZCOPY_AUTO_LOGIN_TYPE'] = '' - os.environ['AZCOPY_SPA_APPLICATION_ID'] = '' - os.environ['AZCOPY_TENANT_ID'] = '' - - def util_test_blobfs_upload_1Kb_file( - self, - explictFromTo=False): - # create file of size 1KB - filename = "test_blob_1kb_file.txt" - file_path = util.create_test_file(filename, 1024) - # upload the file using Azcopy - cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the file uploaded file - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - fileUrl = util.test_bfs_account_url + filename - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def util_test_blobfs_upload_64MB_file( - self, - explictFromTo=False): - # create test file of size 64MB - filename = "test_blob_64MB_file.txt" - file_path = util.create_test_file(filename, 64*1024*1024) - # Upload the file using Azcopy - cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the file uploaded - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - fileUrl = util.test_bfs_account_url + filename - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def util_test_blobfs_upload_uneven_multiflush_file( - self, - explicitFromTo=False): - # create test file of size 64MB - filename = "test_uneven_multiflush_64MB_file.txt" - file_path = util.create_test_file(filename, 64*1024*1024) - # Upload the file using AzCopy @ 1MB blocks, 15 block flushes (5 flushes, 4 15 blocks, 1 4 blocks) - cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("block-size-mb", "1").add_flags("flush-threshold", "15").add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explicitFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the file uploaded - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - fileUrl = util.test_bfs_account_url + filename - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def util_test_blobfs_upload_even_multiflush_file( - self, - explicitFromTo=False): - # create test file of size 64MB - filename = "test_even_multiflush_64MB_file.txt" - file_path = util.create_test_file(filename, 64 * 1024 * 1024) - # Upload the file using AzCopy @ 1MB blocks, 16 block flushes (4 16 block flushes) - cmd = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("block-size-mb", "1").add_flags("flush-threshold", "16").add_flags("log-level", "Info") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explicitFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the file uploaded - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - fileUrl = util.test_bfs_account_url + filename - result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(fileUrl).execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def util_test_blobfs_upload_100_1Kb_file( - self, - explictFromTo=False): - # create dir with 100 1KB files inside it - dir_name = "dir_blobfs_100_1K" - dir_n_file_path = util.create_test_n_files(1024, 100, dir_name) - - # Upload the directory with 100 files inside it - cmd = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_account_url). \ - add_flags("log-level", "Info").add_flags("recursive","true") - util.process_oauth_command( - cmd, - "LocalBlobFS" if explictFromTo else "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Validate the uploaded directory - os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey - dirUrl = util.test_bfs_account_url + dir_name - result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] - os.environ['ACCOUNT_KEY'] = '' - - def test_blobfs_upload_1Kb_file_with_oauth(self): - self.util_test_blobfs_upload_1Kb_file() - - @unittest.skip("single file authentication for oauth covered by 1kb case") - def test_blobfs_upload_64MB_file_with_oauth(self): - self.util_test_blobfs_upload_64MB_file() - - def test_blobfs_upload_100_1Kb_file_with_oauth(self): - self.util_test_blobfs_upload_100_1Kb_file() - - def test_blobfs_upload_uneven_multiflush_with_oauth(self): - self.util_test_blobfs_upload_uneven_multiflush_file() - - def test_blobfs_upload_even_multiflush_with_oauth(self): - self.util_test_blobfs_upload_even_multiflush_file()