From b85da79737ba28c05e0fba08f12b0fe5c7647357 Mon Sep 17 00:00:00 2001 From: Adele Reed Date: Wed, 3 Jan 2024 11:54:49 -0800 Subject: [PATCH 1/9] Generate a DFS service client when working with Blob as well --- common/util.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/common/util.go b/common/util.go index 077bbffc1..d50f96a30 100644 --- a/common/util.go +++ b/common/util.go @@ -116,7 +116,7 @@ func GetServiceClientForLocation(loc Location, ) (*ServiceClient, error) { ret := &ServiceClient{} switch loc { - case ELocation.BlobFS(): + case ELocation.BlobFS(), ELocation.Blob(): // Since we always may need to interact with DFS while working with Blob, we should just attach both. datalakeURLParts, err := azdatalake.ParseURL(resourceURL) if err != nil { return nil, err @@ -150,9 +150,6 @@ func GetServiceClientForLocation(loc Location, ret.dsc = dsc - // For BlobFS, we additionally create a blob client as well. We interact with both endpoints. - fallthrough - case ELocation.Blob(): blobURLParts, err := blob.ParseURL(resourceURL) if err != nil { return nil, err @@ -162,23 +159,23 @@ func GetServiceClientForLocation(loc Location, // In case we are creating a blob client for a datalake target, correct the endpoint blobURLParts.Host = strings.Replace(blobURLParts.Host, ".dfs", ".blob", 1) resourceURL = blobURLParts.String() - var o *blobservice.ClientOptions + var bso *blobservice.ClientOptions var bsc *blobservice.Client if policyOptions != nil { - o = &blobservice.ClientOptions{ClientOptions: *policyOptions} + bso = &blobservice.ClientOptions{ClientOptions: *policyOptions} } if credType.IsAzureOAuth() { - bsc, err = blobservice.NewClient(resourceURL, cred, o) + bsc, err = blobservice.NewClient(resourceURL, cred, bso) } else if credType.IsSharedKey() { var sharedKeyCred *blob.SharedKeyCredential sharedKeyCred, err = GetBlobSharedKeyCredential() if err != nil { return nil, err } - bsc, err = blobservice.NewClientWithSharedKeyCredential(resourceURL, sharedKeyCred, o) + bsc, err = blobservice.NewClientWithSharedKeyCredential(resourceURL, sharedKeyCred, bso) } else { - bsc, err = blobservice.NewClientWithNoCredential(resourceURL, o) + bsc, err = blobservice.NewClientWithNoCredential(resourceURL, bso) } if err != nil { From 9446ea68b64c9bbaf5215a256ee594c279789629 Mon Sep 17 00:00:00 2001 From: Adele Reed Date: Thu, 4 Jan 2024 13:25:39 -0800 Subject: [PATCH 2/9] Fix testing, default include directory stubs when sensible Also removes leftover isHNStoHNS flag from Sync. --- cmd/copy.go | 4 ++++ cmd/sync.go | 4 ---- cmd/syncEnumerator.go | 6 ++++-- e2etest/declarativeHelpers.go | 8 ++++++-- e2etest/declarativeResourceManagers.go | 3 ++- e2etest/declarativeRunner.go | 13 +++++++------ e2etest/declarativeScenario.go | 13 +++++-------- e2etest/zt_preserve_properties_test.go | 2 +- 8 files changed, 29 insertions(+), 24 deletions(-) diff --git a/cmd/copy.go b/cmd/copy.go index c046d4281..90ea2a1d7 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -686,6 +686,10 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) { cooked.asSubdir = raw.asSubdir cooked.IncludeDirectoryStubs = raw.includeDirectoryStubs + if cooked.preservePermissions.IsTruthy() && cooked.FromTo.From() == common.ELocation.Blob() { + // If a user is trying to persist from Blob storage with ACLs, they probably want directories too, because ACLs only exist in HNS. + cooked.IncludeDirectoryStubs = true + } cooked.backupMode = raw.backupMode if err = validateBackupMode(cooked.backupMode, cooked.FromTo); err != nil { diff --git a/cmd/sync.go b/cmd/sync.go index 3e565ff51..05f3d7119 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -258,9 +258,6 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) { // return cooked, err // } cooked.preservePermissions = common.NewPreservePermissionsOption(isUserPersistingPermissions, raw.preserveOwner, cooked.fromTo) - if cooked.fromTo == common.EFromTo.BlobBlob() && cooked.preservePermissions.IsTruthy() { - cooked.isHNSToHNS = true // override HNS settings, since if a user is tx'ing blob->blob and copying permissions, it's DEFINITELY going to be HNS (since perms don't exist w/o HNS). - } cooked.preservePOSIXProperties = raw.preservePOSIXProperties if cooked.preservePOSIXProperties && !areBothLocationsPOSIXAware(cooked.fromTo) { @@ -380,7 +377,6 @@ type cookedSyncCmdArgs struct { fromTo common.FromTo credentialInfo common.CredentialInfo s2sSourceCredentialType common.CredentialType - isHNSToHNS bool // Because DFS sources and destinations are obscured, this is necessary for folder property transfers on ADLS Gen 2. // filters recursive bool diff --git a/cmd/syncEnumerator.go b/cmd/syncEnumerator.go index d9087d47a..b241f3023 100644 --- a/cmd/syncEnumerator.go +++ b/cmd/syncEnumerator.go @@ -55,12 +55,14 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s } } + includeDirStubs := cca.fromTo.From().SupportsHnsACLs() && cca.fromTo.To().SupportsHnsACLs() && cca.preservePermissions.IsTruthy() + // TODO: enable symlink support in a future release after evaluating the implications // TODO: Consider passing an errorChannel so that enumeration errors during sync can be conveyed to the caller. // GetProperties is enabled by default as sync supports both upload and download. // This property only supports Files and S3 at the moment, but provided that Files sync is coming soon, enable to avoid stepping on Files sync work dest := cca.fromTo.To() - sourceTraverser, err := InitResourceTraverser(cca.source, cca.fromTo.From(), &ctx, &srcCredInfo, common.ESymlinkHandlingType.Skip(), nil, cca.recursive, true, cca.isHNSToHNS, common.EPermanentDeleteOption.None(), func(entityType common.EntityType) { + sourceTraverser, err := InitResourceTraverser(cca.source, cca.fromTo.From(), &ctx, &srcCredInfo, common.ESymlinkHandlingType.Skip(), nil, cca.recursive, true, includeDirStubs, common.EPermanentDeleteOption.None(), func(entityType common.EntityType) { if entityType == common.EEntityType.File() { atomic.AddUint64(&cca.atomicSourceFilesScanned, 1) } @@ -81,7 +83,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s // TODO: enable symlink support in a future release after evaluating the implications // GetProperties is enabled by default as sync supports both upload and download. // This property only supports Files and S3 at the moment, but provided that Files sync is coming soon, enable to avoid stepping on Files sync work - destinationTraverser, err := InitResourceTraverser(cca.destination, cca.fromTo.To(), &ctx, &dstCredInfo, common.ESymlinkHandlingType.Skip(), nil, cca.recursive, true, cca.isHNSToHNS, common.EPermanentDeleteOption.None(), func(entityType common.EntityType) { + destinationTraverser, err := InitResourceTraverser(cca.destination, cca.fromTo.To(), &ctx, &dstCredInfo, common.ESymlinkHandlingType.Skip(), nil, cca.recursive, true, includeDirStubs, common.EPermanentDeleteOption.None(), func(entityType common.EntityType) { if entityType == common.EEntityType.File() { atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1) } diff --git a/e2etest/declarativeHelpers.go b/e2etest/declarativeHelpers.go index 6d79010e5..e9f35ab6b 100644 --- a/e2etest/declarativeHelpers.go +++ b/e2etest/declarativeHelpers.go @@ -460,6 +460,11 @@ func (tft TestFromTo) getValues(op Operation) []common.FromTo { if op == eOperation.Sync() { switch fromTo { case common.EFromTo.BlobBlob(), + common.EFromTo.BlobFSBlob(), + common.EFromTo.BlobBlobFS(), + common.EFromTo.BlobFSBlobFS(), + common.EFromTo.BlobFSLocal(), + common.EFromTo.LocalBlobFS(), common.EFromTo.FileFile(), common.EFromTo.LocalBlob(), common.EFromTo.BlobLocal(), @@ -475,8 +480,7 @@ func (tft TestFromTo) getValues(op Operation) []common.FromTo { // TODO: remove this temp block // temp - if fromTo.From() == common.ELocation.S3() || - fromTo.From() == common.ELocation.BlobFS() || fromTo.To() == common.ELocation.BlobFS() { + if fromTo.From() == common.ELocation.S3() { continue // until we implement the declarativeResourceManagers } diff --git a/e2etest/declarativeResourceManagers.go b/e2etest/declarativeResourceManagers.go index 4e8421a46..5e3a74dc5 100644 --- a/e2etest/declarativeResourceManagers.go +++ b/e2etest/declarativeResourceManagers.go @@ -212,6 +212,7 @@ func (r *resourceLocal) createSourceSnapshot(a asserter) { type resourceBlobContainer struct { accountType AccountType + isBlobFS bool containerClient *container.Client rawSasURL *url.URL } @@ -383,7 +384,7 @@ func (r *resourceBlobContainer) getParam(a asserter, stripTopDir, withSas bool, uri = bURLParts.String() } - if r.accountType == EAccountType.HierarchicalNamespaceEnabled() { + if r.isBlobFS { uri = strings.ReplaceAll(uri, "blob", "dfs") } diff --git a/e2etest/declarativeRunner.go b/e2etest/declarativeRunner.go index ade796d67..75b9e8906 100644 --- a/e2etest/declarativeRunner.go +++ b/e2etest/declarativeRunner.go @@ -111,16 +111,16 @@ func RunScenarios( validate Validate, // TODO: do we really want the test author to have to nominate which validation should happen? Pros: better perf of tests. Cons: they have to tell us, and if they tell us wrong test may not test what they think it tests // _ interface{}, // TODO if we want it??, blockBlobsOnly or specific/all blob types -// It would be a pain to list out every combo by hand, -// In addition to the fact that not every credential type is sensible. -// Thus, the E2E framework takes in a requested set of credential types, and applies them where sensible. -// This allows you to make tests use OAuth only, SAS only, etc. + // It would be a pain to list out every combo by hand, + // In addition to the fact that not every credential type is sensible. + // Thus, the E2E framework takes in a requested set of credential types, and applies them where sensible. + // This allows you to make tests use OAuth only, SAS only, etc. requestedCredentialTypesSrc []common.CredentialType, requestedCredentialTypesDst []common.CredentialType, p params, hs *hooks, fs testFiles, -// TODO: do we need something here to explicitly say that we expect success or failure? For now, we are just inferring that from the elements of sourceFiles + // TODO: do we need something here to explicitly say that we expect success or failure? For now, we are just inferring that from the elements of sourceFiles destAccountType AccountType, srcAccountType AccountType, scenarioSuffix string) { @@ -140,8 +140,9 @@ func RunScenarios( } seenFromTos := make(map[common.FromTo]bool) + fromTos := testFromTo.getValues(op) - for _, fromTo := range testFromTo.getValues(op) { + for _, fromTo := range fromTos { // dedupe the scenarios if _, ok := seenFromTos[fromTo]; ok { continue diff --git a/e2etest/declarativeScenario.go b/e2etest/declarativeScenario.go index 78a492138..176046db3 100644 --- a/e2etest/declarativeScenario.go +++ b/e2etest/declarativeScenario.go @@ -107,13 +107,13 @@ func (s *scenario) Run() { // setup scenario // First, validate the accounts make sense for the source/dests if s.srcAccountType.IsBlobOnly() { - s.a.Assert(s.fromTo.From(), equals(), common.ELocation.Blob()) + s.a.Assert(true, equals(), s.fromTo.From() == common.ELocation.Blob() || s.fromTo.From() == common.ELocation.BlobFS()) } if s.destAccountType.IsBlobOnly() { s.a.Assert(s.destAccountType, notEquals(), EAccountType.StdManagedDisk(), "Upload is not supported in MD testing yet") s.a.Assert(s.destAccountType, notEquals(), EAccountType.OAuthManagedDisk(), "Upload is not supported in MD testing yet") - s.a.Assert(s.fromTo.To(), equals(), common.ELocation.Blob()) + s.a.Assert(true, equals(), s.fromTo.From() == common.ELocation.Blob() || s.fromTo.From() == common.ELocation.BlobFS()) } // setup @@ -261,7 +261,7 @@ func (s *scenario) assignSourceAndDest() { return &resourceLocal{common.Iff[string](s.p.destNull && !isSourceAcc, common.Dev_Null, "")} case common.ELocation.File(): return &resourceAzureFileShare{accountType: accType} - case common.ELocation.Blob(): + case common.ELocation.Blob(), common.ELocation.BlobFS(): // TODO: handle the multi-container (whole account) scenario // TODO: handle wider variety of account types if accType.IsManagedDisk() { @@ -270,10 +270,7 @@ func (s *scenario) assignSourceAndDest() { return &resourceManagedDisk{config: *mdCfg} } - return &resourceBlobContainer{accountType: accType} - case common.ELocation.BlobFS(): - s.a.Error("Not implemented yet for blob FS") - return &resourceDummy{} + return &resourceBlobContainer{accountType: accType, isBlobFS: loc == common.ELocation.BlobFS()} case common.ELocation.S3(): s.a.Error("Not implemented yet for S3") return &resourceDummy{} @@ -484,7 +481,7 @@ func (s *scenario) getTransferInfo() (srcRoot string, dstRoot string, expectFold expectFolders = (s.fromTo.From().IsFolderAware() && s.fromTo.To().IsFolderAware() && s.p.allowsFolderTransfers()) || - (s.p.preserveSMBPermissions && s.FromTo() == common.EFromTo.BlobBlob()) || + (s.p.preserveSMBPermissions && s.FromTo().From().SupportsHnsACLs() && s.FromTo().To().SupportsHnsACLs()) || (s.p.preservePOSIXProperties && (s.FromTo() == common.EFromTo.LocalBlob() || s.FromTo() == common.EFromTo.BlobBlob() || s.FromTo() == common.EFromTo.BlobLocal())) expectRootFolder := expectFolders diff --git a/e2etest/zt_preserve_properties_test.go b/e2etest/zt_preserve_properties_test.go index b5301d3d2..e05c8e44d 100644 --- a/e2etest/zt_preserve_properties_test.go +++ b/e2etest/zt_preserve_properties_test.go @@ -57,7 +57,7 @@ func TestProperties_NameValueMetadataCanBeUploaded(t *testing.T) { } func TestProperties_HNSACLs(t *testing.T) { - RunScenarios(t, eOperation.CopyAndSync(), eTestFromTo.Other(common.EFromTo.BlobBlob()), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ + RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.BlobBlob(), common.EFromTo.BlobFSBlobFS(), common.EFromTo.BlobBlobFS(), common.EFromTo.BlobFSBlob()), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ recursive: true, preserveSMBPermissions: true, // this flag is deprecated, but still held over to avoid breaking. }, nil, testFiles{ From c290c31e44202e6d1acafca2b1b3ee4e4e65da0a Mon Sep 17 00:00:00 2001 From: Adele Reed Date: Tue, 9 Jan 2024 13:19:48 -0800 Subject: [PATCH 3/9] Use HNS acct with BlobFS location --- e2etest/declarativeRunner.go | 10 ++++++++++ e2etest/declarativeScenario.go | 2 +- e2etest/zt_remove_test.go | 3 ++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/e2etest/declarativeRunner.go b/e2etest/declarativeRunner.go index 75b9e8906..7278187e4 100644 --- a/e2etest/declarativeRunner.go +++ b/e2etest/declarativeRunner.go @@ -167,6 +167,16 @@ func RunScenarios( subtestName += "-" + scenarioSuffix } + if fromTo.From() == common.ELocation.BlobFS() { + // switch to an account made for dfs + srcAccountType = EAccountType.HierarchicalNamespaceEnabled() + } + + if fromTo.To() == common.ELocation.BlobFS() { + // switch to an account made for dfs + destAccountType = EAccountType.HierarchicalNamespaceEnabled() + } + s := scenario{ srcAccountType: srcAccountType, destAccountType: destAccountType, diff --git a/e2etest/declarativeScenario.go b/e2etest/declarativeScenario.go index 176046db3..abec9f9a5 100644 --- a/e2etest/declarativeScenario.go +++ b/e2etest/declarativeScenario.go @@ -110,7 +110,7 @@ func (s *scenario) Run() { s.a.Assert(true, equals(), s.fromTo.From() == common.ELocation.Blob() || s.fromTo.From() == common.ELocation.BlobFS()) } - if s.destAccountType.IsBlobOnly() { + if s.destAccountType.IsManagedDisk() { s.a.Assert(s.destAccountType, notEquals(), EAccountType.StdManagedDisk(), "Upload is not supported in MD testing yet") s.a.Assert(s.destAccountType, notEquals(), EAccountType.OAuthManagedDisk(), "Upload is not supported in MD testing yet") s.a.Assert(true, equals(), s.fromTo.From() == common.ELocation.Blob() || s.fromTo.From() == common.ELocation.BlobFS()) diff --git a/e2etest/zt_remove_test.go b/e2etest/zt_remove_test.go index 1ede9ed8d..cb22c7ed7 100644 --- a/e2etest/zt_remove_test.go +++ b/e2etest/zt_remove_test.go @@ -39,7 +39,8 @@ func TestRemove_IncludeAfter(t *testing.T) { folder("fold1"), f("fold1/fileb"), } - RunScenarios(t, eOperation.Remove(), eTestFromTo.AllRemove(), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ + // these filters aren't supported for blobFS + RunScenarios(t, eOperation.Remove(), eTestFromTo.Other(common.EFromTo.BlobTrash(), common.EFromTo.FileTrash()), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ recursive: true, }, &hooks{ beforeRunJob: func(h hookHelper) { From d4a520df52aec90b38ebd7835bf72a53c6c76d9e Mon Sep 17 00:00:00 2001 From: Adele Reed Date: Tue, 9 Jan 2024 14:28:53 -0800 Subject: [PATCH 4/9] Adapt to using HNS account instead when using BlobFS --- cmd/zc_traverser_blob.go | 5 +++-- e2etest/declarativeRunner.go | 9 +++++---- e2etest/zt_basic_copy_sync_remove_test.go | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cmd/zc_traverser_blob.go b/cmd/zc_traverser_blob.go index 42a43dd46..424585fff 100644 --- a/cmd/zc_traverser_blob.go +++ b/cmd/zc_traverser_blob.go @@ -256,8 +256,9 @@ func (t *blobTraverser) Traverse(preprocessor objectMorpher, processor objectPro if !t.includeDeleted && (isBlob || err != nil) { return err } - } else if blobURLParts.BlobName == "" && t.preservePermissions.IsTruthy() { - // if the root is a container and we're copying "folders", we should persist the ACLs there too. + } else if blobURLParts.BlobName == "" && (t.preservePermissions.IsTruthy() || t.isDFS) { + // If the root is a container and we're copying "folders", we should persist the ACLs there too. + // For DFS, we should always include the container root. if azcopyScanningLogger != nil { azcopyScanningLogger.Log(common.LogDebug, "Detected the root as a container.") } diff --git a/e2etest/declarativeRunner.go b/e2etest/declarativeRunner.go index 7278187e4..3775e6017 100644 --- a/e2etest/declarativeRunner.go +++ b/e2etest/declarativeRunner.go @@ -167,19 +167,20 @@ func RunScenarios( subtestName += "-" + scenarioSuffix } + usedSrc, usedDst := srcAccountType, destAccountType if fromTo.From() == common.ELocation.BlobFS() { // switch to an account made for dfs - srcAccountType = EAccountType.HierarchicalNamespaceEnabled() + usedSrc = EAccountType.HierarchicalNamespaceEnabled() } if fromTo.To() == common.ELocation.BlobFS() { // switch to an account made for dfs - destAccountType = EAccountType.HierarchicalNamespaceEnabled() + usedDst = EAccountType.HierarchicalNamespaceEnabled() } s := scenario{ - srcAccountType: srcAccountType, - destAccountType: destAccountType, + srcAccountType: usedSrc, + destAccountType: usedDst, subtestName: subtestName, compactScenarioName: compactScenarioName, fullScenarioName: fullScenarioName, diff --git a/e2etest/zt_basic_copy_sync_remove_test.go b/e2etest/zt_basic_copy_sync_remove_test.go index f45ff72b9..add9d0829 100644 --- a/e2etest/zt_basic_copy_sync_remove_test.go +++ b/e2etest/zt_basic_copy_sync_remove_test.go @@ -41,7 +41,7 @@ import ( // ================================ Copy And Sync: Upload, Download, and S2S ========================================= func TestBasic_CopyUploadSingleBlob(t *testing.T) { - RunScenarios(t, eOperation.CopyAndSync(), eTestFromTo.AllUploads(), eValidate.AutoPlusContent(), anonymousAuthOnly, allCredentialTypes, params{ + RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.LocalBlobFS()), eValidate.AutoPlusContent(), anonymousAuthOnly, anonymousAuthOnly, params{ recursive: true, }, nil, testFiles{ defaultSize: "1K", @@ -138,7 +138,7 @@ func TestBasic_CopyUploadLargeAppendBlobBlockSizeFlag(t *testing.T) { } func TestBasic_CopyDownloadSingleBlob(t *testing.T) { - RunScenarios(t, eOperation.CopyAndSync(), eTestFromTo.AllDownloads(), eValidate.Auto(), allCredentialTypes, anonymousAuthOnly, params{ + RunScenarios(t, eOperation.CopyAndSync(), eTestFromTo.AllDownloads(), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ recursive: true, }, nil, testFiles{ defaultSize: "1K", From 80fe144373ff1c2703f0a262aac77d7337d3e406 Mon Sep 17 00:00:00 2001 From: Adele Reed Date: Thu, 22 Feb 2024 18:38:40 -0800 Subject: [PATCH 5/9] Fix tests --- e2etest/zt_basic_cli_ps_auth_test.go | 189 ++++++++++++++++++++-- e2etest/zt_basic_copy_sync_remove_test.go | 22 ++- 2 files changed, 189 insertions(+), 22 deletions(-) diff --git a/e2etest/zt_basic_cli_ps_auth_test.go b/e2etest/zt_basic_cli_ps_auth_test.go index c9fcf8572..4bbcd31cd 100644 --- a/e2etest/zt_basic_cli_ps_auth_test.go +++ b/e2etest/zt_basic_cli_ps_auth_test.go @@ -1,22 +1,179 @@ -// Copyright © Microsoft +// // Copyright © Microsoft +// // +// // Permission is hereby granted, free of charge, to any person obtaining a copy +// // of this software and associated documentation files (the "Software"), to deal +// // in the Software without restriction, including without limitation the rights +// // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// // copies of the Software, and to permit persons to whom the Software is +// // furnished to do so, subject to the following conditions: +// // +// // The above copyright notice and this permission notice shall be included in +// // all copies or substantial portions of the Software. +// // +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// // THE SOFTWARE. +package e2etest + +// +//import ( +// "fmt" +// "os" +// "os/exec" +// "testing" +// +// "github.com/Azure/azure-storage-azcopy/v10/common" +//) +// +//// Purpose: Tests AZCLI and powershell auth tests +// +//func TestBasic_AzCLIAuth(t *testing.T) { +// RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.BlobBlob()), eValidate.Auto(), oAuthOnly, oAuthOnly, params{ // Pass flag values that the test requires. The params struct is a superset of Copy and Sync params +// recursive: true, +// }, &hooks{ +// beforeTestRun: func(h hookHelper) { +// tenId, appId, clientSecret := GlobalInputManager{}.GetServicePrincipalAuth() +// args := []string{ +// "login", +// "--service-principal", +// "-u=" + appId, +// "-p=" + clientSecret, +// } +// if tenId != "" { +// args = append(args, "--tenant="+tenId) +// } +// +// out, err := exec.Command("az", args...).Output() +// if err != nil { +// e := err.(*exec.ExitError) +// t.Logf(string(e.Stderr)) +// t.Logf(string(out)) +// t.Logf("Failed to login with AzCLI " + err.Error()) +// t.FailNow() +// } +// os.Setenv("AZCOPY_AUTO_LOGIN_TYPE", "AZCLI") +// }, +// }, testFiles{ +// defaultSize: "1K", +// shouldTransfer: []interface{}{ +// "wantedfile", +// folder("sub/subsub"), +// "sub/subsub/filea", +// "sub/subsub/filec", +// }, +// }, EAccountType.Standard(), EAccountType.Standard(), "") +//} +// +//func TestBasic_AzCLIAuthLowerCase(t *testing.T) { +// RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.BlobBlob()), eValidate.Auto(), oAuthOnly, oAuthOnly, params{ // Pass flag values that the test requires. The params struct is a superset of Copy and Sync params +// recursive: true, +// }, &hooks{ +// beforeTestRun: func(h hookHelper) { +// tenId, appId, clientSecret := GlobalInputManager{}.GetServicePrincipalAuth() +// args := []string{ +// "login", +// "--service-principal", +// "-u=" + appId, +// "-p=" + clientSecret, +// } +// if tenId != "" { +// args = append(args, "--tenant="+tenId) +// } +// +// out, err := exec.Command("az", args...).Output() +// if err != nil { +// e := err.(*exec.ExitError) +// t.Logf(string(e.Stderr)) +// t.Logf(string(out)) +// t.Logf("Failed to login with AzCLI " + err.Error()) +// t.FailNow() +// } +// os.Setenv("AZCOPY_AUTO_LOGIN_TYPE", "azcli") +// }, +// }, testFiles{ +// defaultSize: "1K", +// shouldTransfer: []interface{}{ +// "wantedfile", +// folder("sub/subsub"), +// "sub/subsub/filea", +// "sub/subsub/filec", +// }, +// }, EAccountType.Standard(), EAccountType.Standard(), "") +//} +// +//func TestBasic_PSAuth(t *testing.T) { +// RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.BlobBlob()), eValidate.Auto(), oAuthOnly, oAuthOnly, params{ // Pass flag values that the test requires. The params struct is a superset of Copy and Sync params +// recursive: true, +// }, &hooks{ +// beforeTestRun: func(h hookHelper) { +// tenId, appId, clientSecret := GlobalInputManager{}.GetServicePrincipalAuth() +// cmd := `$secret = ConvertTo-SecureString -String %s -AsPlainText -Force; +// $cred = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList %s, $secret; +// Connect-AzAccount -ServicePrincipal -Credential $cred` +// if tenId != "" { +// cmd += " -Tenant " + tenId +// } // -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: +// script := fmt.Sprintf(cmd, clientSecret, appId) +// out, err := exec.Command("pwsh", "-Command", script).Output() +// if err != nil { +// e := err.(*exec.ExitError) +// t.Logf(string(e.Stderr)) +// t.Logf(string(out)) +// t.Logf("Failed to login with Powershell " + err.Error()) +// t.FailNow() +// } +// os.Setenv("AZCOPY_AUTO_LOGIN_TYPE", "PSCRED") +// }, +// }, testFiles{ +// defaultSize: "1K", +// shouldTransfer: []interface{}{ +// "wantedfile", +// folder("sub/subsub"), +// "sub/subsub/filea", +// "sub/subsub/filec", +// }, +// }, EAccountType.Standard(), EAccountType.Standard(), "") +//} // -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. +//func TestBasic_PSAuthCamelCase(t *testing.T) { +// RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.BlobBlob()), eValidate.Auto(), oAuthOnly, oAuthOnly, params{ // Pass flag values that the test requires. The params struct is a superset of Copy and Sync params +// recursive: true, +// }, &hooks{ +// beforeTestRun: func(h hookHelper) { +// tenId, appId, clientSecret := GlobalInputManager{}.GetServicePrincipalAuth() +// cmd := `$secret = ConvertTo-SecureString -String %s -AsPlainText -Force; +// $cred = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList %s, $secret; +// Connect-AzAccount -ServicePrincipal -Credential $cred` +// if tenId != "" { +// cmd += " -Tenant " + tenId +// } // -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. +// script := fmt.Sprintf(cmd, clientSecret, appId) +// out, err := exec.Command("pwsh", "-Command", script).Output() +// if err != nil { +// e := err.(*exec.ExitError) +// t.Logf(string(e.Stderr)) +// t.Logf(string(out)) +// t.Logf("Failed to login with Powershell " + err.Error()) +// t.FailNow() +// } +// os.Setenv("AZCOPY_AUTO_LOGIN_TYPE", "PsCred") +// }, +// }, testFiles{ +// defaultSize: "1K", +// shouldTransfer: []interface{}{ +// "wantedfile", +// folder("sub/subsub"), +// "sub/subsub/filea", +// "sub/subsub/filec", +// }, +// }, EAccountType.Standard(), EAccountType.Standard(), "") +//} package e2etest diff --git a/e2etest/zt_basic_copy_sync_remove_test.go b/e2etest/zt_basic_copy_sync_remove_test.go index add9d0829..b95a10b70 100644 --- a/e2etest/zt_basic_copy_sync_remove_test.go +++ b/e2etest/zt_basic_copy_sync_remove_test.go @@ -368,7 +368,7 @@ func TestBasic_CopyRemoveFolderHNS(t *testing.T) { desc: "AllRemove", useAllTos: true, froms: []common.Location{ - common.ELocation.Blob(), // blobfs isn't technically supported; todo: support it properly rather than jank through Blob + common.ELocation.BlobFS(), }, tos: []common.Location{ common.ELocation.Unknown(), @@ -419,8 +419,19 @@ func TestBasic_CopyRemoveFolderHNS(t *testing.T) { } func TestBasic_CopyRemoveContainer(t *testing.T) { + bfsRemove := TestFromTo{ + desc: "AllRemove", + useAllTos: true, + froms: []common.Location{ + common.ELocation.Blob(), // If you have a container-level SAS and a HNS account, you can't delete the container. HNS should not be included here. + common.ELocation.File(), + }, + tos: []common.Location{ + common.ELocation.Unknown(), + }, + } - RunScenarios(t, eOperation.Remove(), eTestFromTo.AllRemove(), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ + RunScenarios(t, eOperation.Remove(), bfsRemove, eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ recursive: true, relativeSourcePath: "", }, nil, testFiles{ @@ -438,7 +449,7 @@ func TestBasic_CopyRemoveContainerHNS(t *testing.T) { desc: "AllRemove", useAllTos: true, froms: []common.Location{ - common.ELocation.Blob(), // blobfs isn't technically supported; todo: support it properly rather than jank through Blob + common.ELocation.BlobFS(), }, tos: []common.Location{ common.ELocation.Unknown(), @@ -472,7 +483,6 @@ func TestBasic_CopyRemoveContainerHNS(t *testing.T) { _, err = fsURL.GetAccessControl(ctx, nil) a.Assert(err, notEquals(), nil) a.Assert(datalakeerror.HasCode(err, "FilesystemNotFound"), equals(), true) - }, }, testFiles{ @@ -869,7 +879,7 @@ func TestBasic_OverwriteHNSDirWithChildren(t *testing.T) { RunScenarios( t, eOperation.Copy(), - eTestFromTo.Other(common.EFromTo.LocalBlobFS()), + eTestFromTo.Other(common.EFromTo.BlobFSBlobFS()), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, @@ -1066,7 +1076,7 @@ func TestBasic_SyncRemoveFoldersHNS(t *testing.T) { RunScenarios( t, eOperation.Sync(), - eTestFromTo.Other(common.EFromTo.BlobBlob()), + eTestFromTo.Other(common.EFromTo.BlobFSBlobFS()), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, From 8a15eec1ebe0f567da92f786360ce1ca41aeb9bc Mon Sep 17 00:00:00 2001 From: Adele Reed Date: Fri, 23 Feb 2024 12:07:39 -0800 Subject: [PATCH 6/9] Hopefully final test fixes? --- e2etest/declarativeScenario.go | 16 +++++++++------- e2etest/zt_enumeration_filter_test.go | 15 --------------- e2etest/zt_preserve_properties_test.go | 2 +- ste/sender-blobFS.go | 19 +++++++++++++++---- ste/sender-blobFSFromLocal.go | 7 ++++++- 5 files changed, 31 insertions(+), 28 deletions(-) diff --git a/e2etest/declarativeScenario.go b/e2etest/declarativeScenario.go index abec9f9a5..ef70b9e96 100644 --- a/e2etest/declarativeScenario.go +++ b/e2etest/declarativeScenario.go @@ -580,7 +580,7 @@ func (s *scenario) validateProperties() { // validate all the different things s.validatePOSIXProperties(f, actual.nameValueMetadata) s.validateSymlink(f, actual.nameValueMetadata) - s.validateMetadata(expected.nameValueMetadata, actual.nameValueMetadata) + s.validateMetadata(f, expected.nameValueMetadata, actual.nameValueMetadata) s.validateBlobTags(expected.blobTags, actual.blobTags) s.validateContentHeaders(expected.contentHeaders, actual.contentHeaders) s.validateCreateTime(expected.creationTime, actual.creationTime) @@ -730,20 +730,22 @@ func metadataWithProperCasing(original map[string]*string) map[string]*string { } // // Individual property validation routines -func (s *scenario) validateMetadata(expected, actual map[string]*string) { +func (s *scenario) validateMetadata(f *testObject, expected, actual map[string]*string) { + cased := metadataWithProperCasing(actual) + for _, v := range common.AllLinuxProperties { // properties are evaluated elsewhere delete(expected, v) - delete(actual, v) + delete(cased, v) } - s.a.Assert(len(actual), equals(), len(expected), "Both should have same number of metadata entries") - cased := metadataWithProperCasing(actual) + s.a.Assert(len(cased), equals(), len(expected), "Both should have same number of metadata entries") + for key := range expected { exValue := expected[key] actualValue, ok := cased[key] - s.a.Assert(ok, equals(), true, fmt.Sprintf("expect key '%s' to be found in destination metadata", key)) + s.a.Assert(ok, equals(), true, fmt.Sprintf("%s: expect key '%s' to be found in destination metadata", f.name, key)) if ok { - s.a.Assert(exValue, equals(), actualValue, fmt.Sprintf("Expect value for key '%s' to be '%s' but found '%s'", key, *exValue, *actualValue)) + s.a.Assert(exValue, equals(), actualValue, fmt.Sprintf("%s: Expect value for key '%s' to be '%s' but found '%s'", f.name, key, *exValue, *actualValue)) } } } diff --git a/e2etest/zt_enumeration_filter_test.go b/e2etest/zt_enumeration_filter_test.go index 19eef997e..a044b69d2 100644 --- a/e2etest/zt_enumeration_filter_test.go +++ b/e2etest/zt_enumeration_filter_test.go @@ -162,21 +162,6 @@ func TestFilter_RemoveFolder(t *testing.T) { }, EAccountType.Standard(), EAccountType.Standard(), "") } -func TestFilter_RemoveContainer(t *testing.T) { - - RunScenarios(t, eOperation.Remove(), eTestFromTo.AllRemove(), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ - recursive: true, - relativeSourcePath: "", - }, nil, testFiles{ - defaultSize: "1K", - shouldTransfer: []interface{}{ - "file1.txt", - "folder1/file11.txt", - "folder1/file12.txt", - }, - }, EAccountType.Standard(), EAccountType.Standard(), "") -} - func TestFilter_ExcludePath(t *testing.T) { RunScenarios(t, eOperation.Copy(), eTestFromTo.AllSourcesToOneDest(), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ recursive: true, diff --git a/e2etest/zt_preserve_properties_test.go b/e2etest/zt_preserve_properties_test.go index e05c8e44d..b90673a5f 100644 --- a/e2etest/zt_preserve_properties_test.go +++ b/e2etest/zt_preserve_properties_test.go @@ -44,7 +44,7 @@ func TestProperties_NameValueMetadataIsPreservedS2S(t *testing.T) { func TestProperties_NameValueMetadataCanBeUploaded(t *testing.T) { expectedMap := map[string]*string{"foo": to.Ptr("abc"), "bar": to.Ptr("def"), "baz": to.Ptr("state=a;b")} - RunScenarios(t, eOperation.Copy(), eTestFromTo.AllUploads(), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ + RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.LocalBlobFS()), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ recursive: true, metadata: "foo=abc;bar=def;baz=state=a\\;b", }, nil, testFiles{ diff --git a/ste/sender-blobFS.go b/ste/sender-blobFS.go index 2e53bec8e..0fa9412e1 100644 --- a/ste/sender-blobFS.go +++ b/ste/sender-blobFS.go @@ -56,6 +56,7 @@ type blobFSSenderBase struct { pacer pacer creationTimeHeaders *file.HTTPHeaders flushThreshold int64 + metadataToSet common.Metadata } func newBlobFSSenderBase(jptm IJobPartTransferMgr, destination string, pacer pacer, sip ISourceInfoProvider) (*blobFSSenderBase, error) { @@ -107,6 +108,7 @@ func newBlobFSSenderBase(jptm IJobPartTransferMgr, destination string, pacer pac pacer: pacer, creationTimeHeaders: &headers, flushThreshold: chunkSize * int64(ADLSFlushThreshold), + metadataToSet: props.SrcMetadata, }, nil } @@ -243,12 +245,12 @@ func (u *blobFSSenderBase) GetSourcePOSIXProperties() (common.UnixStatAdapter, e func (u *blobFSSenderBase) SetPOSIXProperties() error { adapter, err := u.GetSourcePOSIXProperties() if err != nil { - return fmt.Errorf("failed to get POSIX properties") + return fmt.Errorf("failed to get POSIX properties: %w", err) } else if adapter == nil { return nil } - meta := common.Metadata{} + meta := u.metadataToSet common.AddStatToBlobMetadata(adapter, meta) delete(meta, common.POSIXFolderMeta) // Can't be set on HNS accounts. @@ -257,7 +259,16 @@ func (u *blobFSSenderBase) SetPOSIXProperties() error { } func (u *blobFSSenderBase) SetFolderProperties() error { - return u.SetPOSIXProperties() + if u.jptm.Info().PreservePOSIXProperties { + return u.SetPOSIXProperties() + } else if len(u.metadataToSet) > 0 { + _, err := u.blobClient.SetMetadata(u.jptm.Context(), u.metadataToSet, nil) + if err != nil { + return fmt.Errorf("failed to set metadata: %w", err) + } + } + + return nil } func (u *blobFSSenderBase) DirUrlToString() string { @@ -282,7 +293,7 @@ func (u *blobFSSenderBase) SendSymlink(linkData string) error { common.AddStatToBlobMetadata(adapter, meta) meta[common.POSIXSymlinkMeta] = to.Ptr("true") // just in case there isn't any metadata - blobHeaders := blob.HTTPHeaders{ // translate headers, since those still apply + blobHeaders := blob.HTTPHeaders{ // translate headers, since those still apply BlobContentType: u.creationTimeHeaders.ContentType, BlobContentEncoding: u.creationTimeHeaders.ContentEncoding, BlobContentLanguage: u.creationTimeHeaders.ContentLanguage, diff --git a/ste/sender-blobFSFromLocal.go b/ste/sender-blobFSFromLocal.go index d058ad321..3c6e936db 100644 --- a/ste/sender-blobFSFromLocal.go +++ b/ste/sender-blobFSFromLocal.go @@ -97,11 +97,16 @@ func (u *blobFSUploader) Epilogue() { // Write POSIX data if jptm.IsLive() { - if jptm.Info().PreservePOSIXProperties { + if jptm.Info().PreservePOSIXProperties { // metadata would be set here err := u.SetPOSIXProperties() if err != nil { jptm.FailActiveUpload("Setting POSIX Properties", err) } + } else if len(u.metadataToSet) > 0 { // but if we aren't writing POSIX properties, let's set metadata to be consistent. + _, err := u.blobClient.SetMetadata(u.jptm.Context(), u.metadataToSet, nil) + if err != nil { + jptm.FailActiveUpload("Setting blob metadata", err) + } } } } From 1df64780d8defa321343aee4e5f0aa9738ae31be Mon Sep 17 00:00:00 2001 From: Adele Reed Date: Fri, 23 Feb 2024 14:53:16 -0800 Subject: [PATCH 7/9] Revert ps cli auth tests --- e2etest/zt_basic_cli_ps_auth_test.go | 190 +++------------------------ 1 file changed, 17 insertions(+), 173 deletions(-) diff --git a/e2etest/zt_basic_cli_ps_auth_test.go b/e2etest/zt_basic_cli_ps_auth_test.go index 4bbcd31cd..6399eb605 100644 --- a/e2etest/zt_basic_cli_ps_auth_test.go +++ b/e2etest/zt_basic_cli_ps_auth_test.go @@ -1,179 +1,23 @@ -// // Copyright © Microsoft -// // -// // Permission is hereby granted, free of charge, to any person obtaining a copy -// // of this software and associated documentation files (the "Software"), to deal -// // in the Software without restriction, including without limitation the rights -// // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// // copies of the Software, and to permit persons to whom the Software is -// // furnished to do so, subject to the following conditions: -// // -// // The above copyright notice and this permission notice shall be included in -// // all copies or substantial portions of the Software. -// // -// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// // THE SOFTWARE. -package e2etest - -// -//import ( -// "fmt" -// "os" -// "os/exec" -// "testing" -// -// "github.com/Azure/azure-storage-azcopy/v10/common" -//) -// -//// Purpose: Tests AZCLI and powershell auth tests -// -//func TestBasic_AzCLIAuth(t *testing.T) { -// RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.BlobBlob()), eValidate.Auto(), oAuthOnly, oAuthOnly, params{ // Pass flag values that the test requires. The params struct is a superset of Copy and Sync params -// recursive: true, -// }, &hooks{ -// beforeTestRun: func(h hookHelper) { -// tenId, appId, clientSecret := GlobalInputManager{}.GetServicePrincipalAuth() -// args := []string{ -// "login", -// "--service-principal", -// "-u=" + appId, -// "-p=" + clientSecret, -// } -// if tenId != "" { -// args = append(args, "--tenant="+tenId) -// } +// Copyright © Microsoft // -// out, err := exec.Command("az", args...).Output() -// if err != nil { -// e := err.(*exec.ExitError) -// t.Logf(string(e.Stderr)) -// t.Logf(string(out)) -// t.Logf("Failed to login with AzCLI " + err.Error()) -// t.FailNow() -// } -// os.Setenv("AZCOPY_AUTO_LOGIN_TYPE", "AZCLI") -// }, -// }, testFiles{ -// defaultSize: "1K", -// shouldTransfer: []interface{}{ -// "wantedfile", -// folder("sub/subsub"), -// "sub/subsub/filea", -// "sub/subsub/filec", -// }, -// }, EAccountType.Standard(), EAccountType.Standard(), "") -//} +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: // -//func TestBasic_AzCLIAuthLowerCase(t *testing.T) { -// RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.BlobBlob()), eValidate.Auto(), oAuthOnly, oAuthOnly, params{ // Pass flag values that the test requires. The params struct is a superset of Copy and Sync params -// recursive: true, -// }, &hooks{ -// beforeTestRun: func(h hookHelper) { -// tenId, appId, clientSecret := GlobalInputManager{}.GetServicePrincipalAuth() -// args := []string{ -// "login", -// "--service-principal", -// "-u=" + appId, -// "-p=" + clientSecret, -// } -// if tenId != "" { -// args = append(args, "--tenant="+tenId) -// } +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. // -// out, err := exec.Command("az", args...).Output() -// if err != nil { -// e := err.(*exec.ExitError) -// t.Logf(string(e.Stderr)) -// t.Logf(string(out)) -// t.Logf("Failed to login with AzCLI " + err.Error()) -// t.FailNow() -// } -// os.Setenv("AZCOPY_AUTO_LOGIN_TYPE", "azcli") -// }, -// }, testFiles{ -// defaultSize: "1K", -// shouldTransfer: []interface{}{ -// "wantedfile", -// folder("sub/subsub"), -// "sub/subsub/filea", -// "sub/subsub/filec", -// }, -// }, EAccountType.Standard(), EAccountType.Standard(), "") -//} -// -//func TestBasic_PSAuth(t *testing.T) { -// RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.BlobBlob()), eValidate.Auto(), oAuthOnly, oAuthOnly, params{ // Pass flag values that the test requires. The params struct is a superset of Copy and Sync params -// recursive: true, -// }, &hooks{ -// beforeTestRun: func(h hookHelper) { -// tenId, appId, clientSecret := GlobalInputManager{}.GetServicePrincipalAuth() -// cmd := `$secret = ConvertTo-SecureString -String %s -AsPlainText -Force; -// $cred = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList %s, $secret; -// Connect-AzAccount -ServicePrincipal -Credential $cred` -// if tenId != "" { -// cmd += " -Tenant " + tenId -// } -// -// script := fmt.Sprintf(cmd, clientSecret, appId) -// out, err := exec.Command("pwsh", "-Command", script).Output() -// if err != nil { -// e := err.(*exec.ExitError) -// t.Logf(string(e.Stderr)) -// t.Logf(string(out)) -// t.Logf("Failed to login with Powershell " + err.Error()) -// t.FailNow() -// } -// os.Setenv("AZCOPY_AUTO_LOGIN_TYPE", "PSCRED") -// }, -// }, testFiles{ -// defaultSize: "1K", -// shouldTransfer: []interface{}{ -// "wantedfile", -// folder("sub/subsub"), -// "sub/subsub/filea", -// "sub/subsub/filec", -// }, -// }, EAccountType.Standard(), EAccountType.Standard(), "") -//} -// -//func TestBasic_PSAuthCamelCase(t *testing.T) { -// RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.BlobBlob()), eValidate.Auto(), oAuthOnly, oAuthOnly, params{ // Pass flag values that the test requires. The params struct is a superset of Copy and Sync params -// recursive: true, -// }, &hooks{ -// beforeTestRun: func(h hookHelper) { -// tenId, appId, clientSecret := GlobalInputManager{}.GetServicePrincipalAuth() -// cmd := `$secret = ConvertTo-SecureString -String %s -AsPlainText -Force; -// $cred = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList %s, $secret; -// Connect-AzAccount -ServicePrincipal -Credential $cred` -// if tenId != "" { -// cmd += " -Tenant " + tenId -// } -// -// script := fmt.Sprintf(cmd, clientSecret, appId) -// out, err := exec.Command("pwsh", "-Command", script).Output() -// if err != nil { -// e := err.(*exec.ExitError) -// t.Logf(string(e.Stderr)) -// t.Logf(string(out)) -// t.Logf("Failed to login with Powershell " + err.Error()) -// t.FailNow() -// } -// os.Setenv("AZCOPY_AUTO_LOGIN_TYPE", "PsCred") -// }, -// }, testFiles{ -// defaultSize: "1K", -// shouldTransfer: []interface{}{ -// "wantedfile", -// folder("sub/subsub"), -// "sub/subsub/filea", -// "sub/subsub/filec", -// }, -// }, EAccountType.Standard(), EAccountType.Standard(), "") -//} +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +package e2etest package e2etest From 2f555e890554907fbf188823fa612dbc10cb1cfe Mon Sep 17 00:00:00 2001 From: Adele Reed Date: Fri, 23 Feb 2024 14:57:20 -0800 Subject: [PATCH 8/9] Accidental leftovers of manual local testing --- e2etest/zt_basic_copy_sync_remove_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2etest/zt_basic_copy_sync_remove_test.go b/e2etest/zt_basic_copy_sync_remove_test.go index b95a10b70..f6418fab8 100644 --- a/e2etest/zt_basic_copy_sync_remove_test.go +++ b/e2etest/zt_basic_copy_sync_remove_test.go @@ -41,7 +41,7 @@ import ( // ================================ Copy And Sync: Upload, Download, and S2S ========================================= func TestBasic_CopyUploadSingleBlob(t *testing.T) { - RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.LocalBlobFS()), eValidate.AutoPlusContent(), anonymousAuthOnly, anonymousAuthOnly, params{ + RunScenarios(t, eOperation.CopyAndSync(), eTestFromTo.AllUploads(), eValidate.AutoPlusContent(), anonymousAuthOnly, anonymousAuthOnly, params{ recursive: true, }, nil, testFiles{ defaultSize: "1K", @@ -419,7 +419,7 @@ func TestBasic_CopyRemoveFolderHNS(t *testing.T) { } func TestBasic_CopyRemoveContainer(t *testing.T) { - bfsRemove := TestFromTo{ + allButBfsRemove := TestFromTo{ desc: "AllRemove", useAllTos: true, froms: []common.Location{ @@ -431,7 +431,7 @@ func TestBasic_CopyRemoveContainer(t *testing.T) { }, } - RunScenarios(t, eOperation.Remove(), bfsRemove, eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ + RunScenarios(t, eOperation.Remove(), allButBfsRemove, eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ recursive: true, relativeSourcePath: "", }, nil, testFiles{ From 6a3956bad2625ef35b5f9d71a432d6260a8f418a Mon Sep 17 00:00:00 2001 From: Adele Reed Date: Mon, 26 Feb 2024 15:20:13 -0800 Subject: [PATCH 9/9] Clean up other test reductions --- e2etest/zt_basic_cli_ps_auth_test.go | 2 -- e2etest/zt_basic_copy_sync_remove_test.go | 4 ++-- e2etest/zt_preserve_properties_test.go | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/e2etest/zt_basic_cli_ps_auth_test.go b/e2etest/zt_basic_cli_ps_auth_test.go index 6399eb605..9d9079f7f 100644 --- a/e2etest/zt_basic_cli_ps_auth_test.go +++ b/e2etest/zt_basic_cli_ps_auth_test.go @@ -19,8 +19,6 @@ // THE SOFTWARE. package e2etest -package e2etest - import ( "testing" diff --git a/e2etest/zt_basic_copy_sync_remove_test.go b/e2etest/zt_basic_copy_sync_remove_test.go index f6418fab8..d8a57da0e 100644 --- a/e2etest/zt_basic_copy_sync_remove_test.go +++ b/e2etest/zt_basic_copy_sync_remove_test.go @@ -41,7 +41,7 @@ import ( // ================================ Copy And Sync: Upload, Download, and S2S ========================================= func TestBasic_CopyUploadSingleBlob(t *testing.T) { - RunScenarios(t, eOperation.CopyAndSync(), eTestFromTo.AllUploads(), eValidate.AutoPlusContent(), anonymousAuthOnly, anonymousAuthOnly, params{ + RunScenarios(t, eOperation.CopyAndSync(), eTestFromTo.AllUploads(), eValidate.AutoPlusContent(), anonymousAuthOnly, allCredentialTypes, params{ recursive: true, }, nil, testFiles{ defaultSize: "1K", @@ -138,7 +138,7 @@ func TestBasic_CopyUploadLargeAppendBlobBlockSizeFlag(t *testing.T) { } func TestBasic_CopyDownloadSingleBlob(t *testing.T) { - RunScenarios(t, eOperation.CopyAndSync(), eTestFromTo.AllDownloads(), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ + RunScenarios(t, eOperation.CopyAndSync(), eTestFromTo.AllDownloads(), eValidate.Auto(), allCredentialTypes, anonymousAuthOnly, params{ recursive: true, }, nil, testFiles{ defaultSize: "1K", diff --git a/e2etest/zt_preserve_properties_test.go b/e2etest/zt_preserve_properties_test.go index b90673a5f..4dbe47866 100644 --- a/e2etest/zt_preserve_properties_test.go +++ b/e2etest/zt_preserve_properties_test.go @@ -44,7 +44,7 @@ func TestProperties_NameValueMetadataIsPreservedS2S(t *testing.T) { func TestProperties_NameValueMetadataCanBeUploaded(t *testing.T) { expectedMap := map[string]*string{"foo": to.Ptr("abc"), "bar": to.Ptr("def"), "baz": to.Ptr("state=a;b")} - RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.LocalBlobFS()), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ + RunScenarios(t, eOperation.Copy(), eTestFromTo.AllUploads(), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ recursive: true, metadata: "foo=abc;bar=def;baz=state=a\\;b", }, nil, testFiles{ @@ -57,7 +57,7 @@ func TestProperties_NameValueMetadataCanBeUploaded(t *testing.T) { } func TestProperties_HNSACLs(t *testing.T) { - RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.BlobBlob(), common.EFromTo.BlobFSBlobFS(), common.EFromTo.BlobBlobFS(), common.EFromTo.BlobFSBlob()), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ + RunScenarios(t, eOperation.CopyAndSync(), eTestFromTo.Other(common.EFromTo.BlobBlob(), common.EFromTo.BlobFSBlobFS(), common.EFromTo.BlobBlobFS(), common.EFromTo.BlobFSBlob()), eValidate.Auto(), anonymousAuthOnly, anonymousAuthOnly, params{ recursive: true, preserveSMBPermissions: true, // this flag is deprecated, but still held over to avoid breaking. }, nil, testFiles{