Fixed treatment of SAS in new e2e test framework to include container and path names

azure-pipelines.yml

@@ -127,7 +127,7 @@ jobs:
           display_name: "Linux"
           go_path: '$(go env GOPATH)/bin/'
           suffix: ''
-          run_e2e: 'go test -timeout=1h -v ./e2etest | tee test.txt'
+          run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt'
           output_test_logs: ''
           save_exit_code: 'exit_code=${PIPESTATUS[0]}'
           return_exit_code: 'exit "$exit_code"'
@@ -138,7 +138,7 @@ jobs:
           type: 'windows'
           go_path: 'C:\Users\VssAdministrator\go\bin\'
           suffix: '.exe'
-          run_e2e: 'go test -timeout=1h -v ./e2etest > test.txt'
+          run_e2e: 'go test -timeout=2h -v ./e2etest > test.txt'
           output_test_logs: 'cat test.txt'
           save_exit_code: 'set exit_code=%errorlevel%'
           return_exit_code: 'exit %exit_code%'
@@ -148,7 +148,7 @@ jobs:
           display_name: "MacOS"
           go_path: '$(go env GOPATH)/bin/'
           suffix: ''
-          run_e2e: 'go test -timeout=1h -v ./e2etest | tee test.txt'
+          run_e2e: 'go test -timeout=2h -v ./e2etest | tee test.txt'
           output_test_logs: ''
           save_exit_code: 'exit_code=${PIPESTATUS[0]}'
           return_exit_code: 'exit "$exit_code"'

e2etest/newe2e_resource_manager_azstorage.go

@@ -2,13 +2,15 @@ package e2etest
 
 import (
 	"fmt"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	blobsas "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
 	blobservice "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
 	blobfscommon "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake"
-	datalakeSAS "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/sas"
+	datalakesas "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/sas"
 	blobfsservice "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/service"
 	filesas "github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/sas"
 	fileservice "github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/service"
+	"github.com/Azure/azure-storage-azcopy/v10/cmd"
 	"github.com/Azure/azure-storage-azcopy/v10/common"
 )
 
@@ -20,7 +22,7 @@ type AzureAccountResourceManager struct {
 	armClient *ARMStorageAccount
 }
 
-func (acct *AzureAccountResourceManager) ApplySAS(URI string, loc common.Location, optList ...GetURIOptions) string {
+func (acct *AzureAccountResourceManager) ApplySAS(URI string, loc common.Location, lev cmd.LocationLevel, et common.EntityType, optList ...GetURIOptions) string {
 	if acct == nil {
 		panic("Account must not be nil to generate a SAS token.")
 	}
@@ -30,47 +32,88 @@ func (acct *AzureAccountResourceManager) ApplySAS(URI string, loc common.Locatio
 		return URI
 	}
 
-	sasVals := opts.AzureOpts.SASValues
-	if sasVals == nil {
+	var sasVals GenericSignatureValues
+	if opts.AzureOpts.SASValues == nil {
+		// Default to account level SAS to cover all our bases
+		sasVals = GenericAccountSignatureValues{}
+	} else {
+		sasVals = opts.AzureOpts.SASValues
 
-		sasVals = GenericServiceSignatureValues{}
 	}
 
 	switch loc {
 	case common.ELocation.Blob():
-		skc, err := blobservice.NewSharedKeyCredential(acct.accountName, acct.accountKey)
+		parts, err := blobsas.ParseURL(URI)
 		common.PanicIfErr(err)
 
-		p, err := sasVals.AsBlob().SignWithSharedKey(skc)
+		skc, err := blobservice.NewSharedKeyCredential(acct.accountName, acct.accountKey)
 		common.PanicIfErr(err)
 
-		parts, err := blobsas.ParseURL(URI)
+		var p blobsas.QueryParameters
+		if _, ok := sasVals.(GenericServiceSignatureValues); ok {
+			v := sasVals.(GenericServiceSignatureValues)
+			v.Level = common.Iff(v.Level == nil, to.Ptr(lev), v.Level)
+			v.ContainerName = parts.ContainerName
+			v.ObjectName = parts.BlobName
+
+			p, err = v.AsBlob().SignWithSharedKey(skc)
+		} else {
+			p, err = sasVals.AsBlob().SignWithSharedKey(skc)
+		}
 		common.PanicIfErr(err)
 
 		parts.SAS = p
 		parts.Scheme = common.Iff(opts.RemoteOpts.Scheme != "", opts.RemoteOpts.Scheme, "https")
 		return parts.String()
 	case common.ELocation.File():
-		skc, err := fileservice.NewSharedKeyCredential(acct.accountName, acct.accountKey)
+		parts, err := filesas.ParseURL(URI)
 		common.PanicIfErr(err)
 
-		p, err := sasVals.AsFile().SignWithSharedKey(skc)
+		skc, err := fileservice.NewSharedKeyCredential(acct.accountName, acct.accountKey)
 		common.PanicIfErr(err)
 
-		parts, err := filesas.ParseURL(URI)
+		var p filesas.QueryParameters
+		if _, ok := sasVals.(GenericServiceSignatureValues); ok {
+			v := sasVals.(GenericServiceSignatureValues)
+			v.Level = common.Iff(v.Level == nil, to.Ptr(lev), v.Level)
+			v.ContainerName = parts.ShareName
+			if et == common.EEntityType.Folder() {
+				v.DirectoryPath = parts.DirectoryOrFilePath
+			} else {
+				v.ObjectName = parts.DirectoryOrFilePath
+			}
+
+			p, err = v.AsFile().SignWithSharedKey(skc)
+		} else {
+			p, err = sasVals.AsFile().SignWithSharedKey(skc)
+		}
 		common.PanicIfErr(err)
 
 		parts.SAS = p
 		parts.Scheme = common.Iff(opts.RemoteOpts.Scheme != "", opts.RemoteOpts.Scheme, "https")
 		return parts.String()
 	case common.ELocation.BlobFS():
-		skc, err := blobfscommon.NewSharedKeyCredential(acct.accountName, acct.accountKey)
+		parts, err := datalakesas.ParseURL(URI)
 		common.PanicIfErr(err)
 
-		p, err := sasVals.AsDatalake().SignWithSharedKey(skc)
+		skc, err := blobfscommon.NewSharedKeyCredential(acct.accountName, acct.accountKey)
 		common.PanicIfErr(err)
 
-		parts, err := datalakeSAS.ParseURL(URI)
+		var p datalakesas.QueryParameters
+		if _, ok := sasVals.(GenericServiceSignatureValues); ok {
+			v := sasVals.(GenericServiceSignatureValues)
+			v.Level = common.Iff(v.Level == nil, to.Ptr(lev), v.Level)
+			v.ContainerName = parts.FileSystemName
+			if et == common.EEntityType.Folder() {
+				v.DirectoryPath = parts.PathName
+			} else {
+				v.ObjectName = parts.PathName
+			}
+
+			p, err = v.AsDatalake().SignWithSharedKey(skc)
+		} else {
+			p, err = sasVals.AsDatalake().SignWithSharedKey(skc)
+		}
 		common.PanicIfErr(err)
 
 		parts.SAS = p

e2etest/newe2e_resource_manager_interface.go

@@ -24,6 +24,8 @@ type RemoteURIOpts struct {
 type AzureURIOpts struct {
 	// Must be manually specified
 	WithSAS bool
+	// Location level for SAS token, defaults to the resource's location level.
+	Level cmd.LocationLevel
 	// Defaults to a resource-level specific minimally permissioned SAS token.
 	SASValues GenericSignatureValues
 }

e2etest/newe2e_resource_managers_blob.go

@@ -134,7 +134,7 @@ func (b *BlobServiceResourceManager) ListContainers(a Asserter) []string {
 
 func (b *BlobServiceResourceManager) URI(opts ...GetURIOptions) string {
 	base := blobStripSAS(b.internalClient.URL())
-	base = b.internalAccount.ApplySAS(base, b.Location(), opts...)
+	base = b.internalAccount.ApplySAS(base, b.Location(), b.Level(), common.EEntityType, opts...)
 
 	return base
 }
@@ -342,7 +342,7 @@ func (b *BlobContainerResourceManager) Level() cmd.LocationLevel {
 
 func (b *BlobContainerResourceManager) URI(opts ...GetURIOptions) string {
 	base := blobStripSAS(b.internalClient.URL())
-	base = b.internalAccount.ApplySAS(base, b.Location(), opts...)
+	base = b.internalAccount.ApplySAS(base, b.Location(), b.Level(), common.EEntityType, opts...)
 
 	return base
 }
@@ -666,7 +666,7 @@ func (b *BlobObjectResourceManager) Level() cmd.LocationLevel {
 
 func (b *BlobObjectResourceManager) URI(opts ...GetURIOptions) string {
 	base := blobStripSAS(b.internalClient.URL())
-	base = b.internalAccount.ApplySAS(base, b.Location(), opts...)
+	base = b.internalAccount.ApplySAS(base, b.Location(), b.Level(), b.EntityType(), opts...)
 
 	return base
 }

e2etest/newe2e_resource_managers_blobfs.go

@@ -81,7 +81,7 @@ func (b *BlobFSServiceResourceManager) Level() cmd.LocationLevel {
 
 func (b *BlobFSServiceResourceManager) URI(opts ...GetURIOptions) string {
 	base := dfsStripSAS(b.internalClient.DFSURL())
-	base = b.internalAccount.ApplySAS(base, b.Location(), opts...)
+	base = b.internalAccount.ApplySAS(base, b.Location(), b.Level(), common.EEntityType, opts...)
 
 	return base
 }
@@ -175,7 +175,7 @@ func (b *BlobFSFileSystemResourceManager) Level() cmd.LocationLevel {
 
 func (b *BlobFSFileSystemResourceManager) URI(opts ...GetURIOptions) string {
 	base := dfsStripSAS(b.internalClient.DFSURL())
-	base = b.internalAccount.ApplySAS(base, b.Location(), opts...)
+	base = b.internalAccount.ApplySAS(base, b.Location(), b.Level(), common.EEntityType, opts...)
 
 	return base
 }
@@ -309,7 +309,7 @@ func (b *BlobFSPathResourceProvider) Level() cmd.LocationLevel {
 
 func (b *BlobFSPathResourceProvider) URI(opts ...GetURIOptions) string {
 	base := dfsStripSAS(b.getFileClient().DFSURL()) // obj type doesn't matter here, URL is the same under the hood
-	base = b.internalAccount.ApplySAS(base, b.Location(), opts...)
+	base = b.internalAccount.ApplySAS(base, b.Location(), b.Level(), b.EntityType(), opts...)
 
 	return base
 }

e2etest/newe2e_resource_managers_file.go

@@ -84,7 +84,7 @@ func (s *FileServiceResourceManager) Level() cmd.LocationLevel {
 
 func (s *FileServiceResourceManager) URI(opts ...GetURIOptions) string {
 	base := fileStripSAS(s.internalClient.URL())
-	base = s.internalAccount.ApplySAS(base, s.Location(), opts...)
+	base = s.internalAccount.ApplySAS(base, s.Location(), s.Level(), common.EEntityType, opts...)
 
 	return base
 }
@@ -180,7 +180,7 @@ func (s *FileShareResourceManager) Level() cmd.LocationLevel {
 
 func (s *FileShareResourceManager) URI(opts ...GetURIOptions) string {
 	base := fileStripSAS(s.internalClient.URL())
-	base = s.internalAccount.ApplySAS(base, s.Location(), opts...)
+	base = s.internalAccount.ApplySAS(base, s.Location(), s.Level(), common.EEntityType, opts...)
 
 	return base
 }
@@ -397,7 +397,7 @@ func (f *FileObjectResourceManager) Level() cmd.LocationLevel {
 
 func (f *FileObjectResourceManager) URI(opts ...GetURIOptions) string {
 	base := fileStripSAS(f.getFileClient().URL()) // restype doesn't matter here, same URL under the hood
-	base = f.internalAccount.ApplySAS(base, f.Location(), opts...)
+	base = f.internalAccount.ApplySAS(base, f.Location(), f.Level(), f.EntityType(), opts...)
 
 	return base
 }

e2etest/newe2e_sas_generation.go

@@ -2,8 +2,9 @@ package e2etest
 
 import (
 	blobsas "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
-	datalakeSAS "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/sas"
+	datalakesas "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/sas"
 	filesas "github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/sas"
+	"github.com/Azure/azure-storage-azcopy/v10/cmd"
 	"github.com/Azure/azure-storage-azcopy/v10/common"
 	"time"
 )
@@ -20,10 +21,10 @@ type FileSignatureValues interface {
 
 // DatalakeSignatureValues makes both account/service signature values generic for output from GenericSignatureValues
 type DatalakeSignatureValues interface {
-	SignWithSharedKey(credential *datalakeSAS.SharedKeyCredential) (datalakeSAS.QueryParameters, error)
+	SignWithSharedKey(credential *datalakesas.SharedKeyCredential) (datalakesas.QueryParameters, error)
 }
 
-// GenericSignatureValues are a set of signature values that are portable to all 3 Azure storage services.
+// GenericSignatureValues are a set of signature values that are portable to all 3 Azure storage services and both service and account level SAS.
 // Distinct limitations of these generic values are documented in the struct.
 type GenericSignatureValues interface {
 	AsBlob() BlobSignatureValues
@@ -50,7 +51,7 @@ type GenericServiceSignatureValues struct {
 	// Blob and Datalake overlap nicely
 	// blobsas.ContainerPermissions blobsas.BlobPermissions
 	// filesas.SharePermissions filesas.FilePermissions
-	// datalakeSAS.FileSystemPermissions datalakeSAS.FilePermissions datalakeSAS.DirectoryPermissions
+	// datalakesas.FileSystemPermissions datalakesas.FilePermissions datalakesas.DirectoryPermissions
 	// If zero, defaults to racwdl (read, add, create, write, delete, list
 	Permissions   string
 	IPRange       blobsas.IPRange
@@ -69,6 +70,8 @@ type GenericServiceSignatureValues struct {
 	UnauthorizedObjectID string
 	// CorrelationID is used on Blob & Datalake
 	CorrelationID string
+
+	Level *cmd.LocationLevel
 }
 
 // withDefaults will never have to be called by
@@ -88,6 +91,13 @@ func (vals GenericServiceSignatureValues) withDefaults() GenericServiceSignature
 func (vals GenericServiceSignatureValues) AsBlob() BlobSignatureValues {
 	s := vals.withDefaults()
 
+	blobName := s.ObjectName
+	directoryName := s.DirectoryPath
+	if s.Level != nil && *s.Level != cmd.ELocationLevel.Object() {
+		blobName = ""
+		directoryName = ""
+	}
+
 	return &blobsas.BlobSignatureValues{
 		Version:              s.Version,
 		Protocol:             s.Protocol,
@@ -98,8 +108,8 @@ func (vals GenericServiceSignatureValues) AsBlob() BlobSignatureValues {
 		IPRange:              s.IPRange,
 		Identifier:           s.Identifier,
 		ContainerName:        s.ContainerName,
-		BlobName:             s.ObjectName,
-		Directory:            s.DirectoryPath,
+		BlobName:             blobName,
+		Directory:            directoryName,
 		CacheControl:         s.CacheControl,
 		ContentDisposition:   s.ContentDisposition,
 		ContentEncoding:      s.ContentEncoding,
@@ -115,6 +125,11 @@ func (vals GenericServiceSignatureValues) AsBlob() BlobSignatureValues {
 func (vals GenericServiceSignatureValues) AsFile() FileSignatureValues {
 	s := vals.withDefaults()
 
+	filePath := common.Iff(s.DirectoryPath != "", s.DirectoryPath, s.ObjectName)
+	if s.Level != nil && *s.Level != cmd.ELocationLevel.Object() {
+		filePath = ""
+	}
+
 	return &filesas.SignatureValues{
 		Version:            s.Version,
 		Protocol:           filesas.Protocol(s.Protocol),
@@ -125,7 +140,7 @@ func (vals GenericServiceSignatureValues) AsFile() FileSignatureValues {
 		IPRange:            filesas.IPRange(s.IPRange),
 		Identifier:         s.Identifier,
 		ShareName:          s.ContainerName,
-		FilePath:           common.Iff(s.DirectoryPath != "", s.DirectoryPath, s.ObjectName),
+		FilePath:           filePath,
 		CacheControl:       s.CacheControl,
 		ContentDisposition: s.ContentDisposition,
 		ContentEncoding:    s.ContentEncoding,
@@ -137,17 +152,24 @@ func (vals GenericServiceSignatureValues) AsFile() FileSignatureValues {
 func (vals GenericServiceSignatureValues) AsDatalake() DatalakeSignatureValues {
 	s := vals.withDefaults()
 
-	return &datalakeSAS.DatalakeSignatureValues{
+	filePath := s.ObjectName
+	directoryPath := s.DirectoryPath
+	if s.Level != nil && *s.Level != cmd.ELocationLevel.Object() {
+		filePath = ""
+		directoryPath = ""
+	}
+
+	return &datalakesas.DatalakeSignatureValues{
 		Version:              s.Version,
-		Protocol:             datalakeSAS.Protocol(s.Protocol),
+		Protocol:             datalakesas.Protocol(s.Protocol),
 		StartTime:            s.StartTime,
 		ExpiryTime:           s.ExpiryTime,
 		Permissions:          s.Permissions,
-		IPRange:              datalakeSAS.IPRange(s.IPRange),
+		IPRange:              datalakesas.IPRange(s.IPRange),
 		Identifier:           s.Identifier,
 		FileSystemName:       s.ContainerName,
-		FilePath:             s.ObjectName,
-		DirectoryPath:        s.DirectoryPath,
+		FilePath:             filePath,
+		DirectoryPath:        directoryPath,
 		CacheControl:         s.CacheControl,
 		ContentDisposition:   s.ContentDisposition,
 		ContentEncoding:      s.ContentEncoding,
@@ -167,10 +189,10 @@ type GenericAccountSignatureValues struct {
 	StartTime time.Time
 	// Defaults to 24hr past StartTime
 	ExpiryTime time.Time
-	// Defaults to racwdl, uses blobsas.AccountPermissions, filesas.AccountPermissions, or datalakeSAS.AccountPermissions
+	// Defaults to racwdl, uses blobsas.AccountPermissions, filesas.AccountPermissions, or datalakesas.AccountPermissions
 	Permissions string
 	IPRange     blobsas.IPRange
-	// defaults to sco, uses blobsas.AccountResourceTypes, filesas.AccountResourceTypes, or datalakeSAS.AccountResourceTypes
+	// defaults to sco, uses blobsas.AccountResourceTypes, filesas.AccountResourceTypes, or datalakesas.AccountResourceTypes
 	ResourceTypes string
 }
 
@@ -219,13 +241,13 @@ func (vals GenericAccountSignatureValues) AsFile() FileSignatureValues {
 func (vals GenericAccountSignatureValues) AsDatalake() DatalakeSignatureValues {
 	s := vals.withDefaults()
 
-	return datalakeSAS.AccountSignatureValues{
+	return datalakesas.AccountSignatureValues{
 		Version:       s.Version,
-		Protocol:      datalakeSAS.Protocol(s.Protocol),
+		Protocol:      datalakesas.Protocol(s.Protocol),
 		StartTime:     s.StartTime,
 		ExpiryTime:    s.ExpiryTime,
 		Permissions:   s.Permissions,
-		IPRange:       datalakeSAS.IPRange(s.IPRange),
+		IPRange:       datalakesas.IPRange(s.IPRange),
 		ResourceTypes: s.ResourceTypes,
 	}
 }