Merge branch 'main' into adreed/e2e-test-inherit-auth

Author: wonwuakpa-msft

.github/CODEOWNERS

@@ -1 +1 @@
-*       @gapra-msft @adreed-msft @dphulkar-msft @vibhansa-msft @seanmcc-msft @souravgupta-msft @tanyasethi-msft
+*       @gapra-msft @adreed-msft @dphulkar-msft @vibhansa-msft @seanmcc-msft @souravgupta-msft @tanyasethi-msft @wonwuakpa-msft

.github/workflows/update-wiki-docs.yml

@@ -1,10 +1,8 @@
 name: Update wiki docs
 
 on:
-  push:
-    branches: [ "main" ]
-  pull_request:
-    branches: [ "main" ]
+  release:
+    types: [published]
 
 concurrency:
   group: wiki

cmd/copy.go

@@ -189,20 +189,6 @@ type rawCopyCmdArgs struct {
 	deleteDestinationFileIfNecessary bool
 }
 
-func (raw *rawCopyCmdArgs) parsePatterns(pattern string) (cookedPatterns []string) {
-	cookedPatterns = make([]string, 0)
-	rawPatterns := strings.Split(pattern, ";")
-	for _, pattern := range rawPatterns {
-
-		// skip the empty patterns
-		if len(pattern) != 0 {
-			cookedPatterns = append(cookedPatterns, pattern)
-		}
-	}
-
-	return
-}
-
 // blocSizeInBytes converts a FLOATING POINT number of MiB, to a number of bytes
 // A non-nil error is returned if the conversion is not possible to do accurately (e.g. it comes out of a fractional number of bytes)
 // The purpose of using floating point is to allow specialist users (e.g. those who want small block sizes to tune their read IOPS)
@@ -226,47 +212,6 @@ func blockSizeInBytes(rawBlockSizeInMiB float64) (int64, error) {
 	return int64(math.Round(rawSizeInBytes)), nil
 }
 
-// returns result of stripping and if striptopdir is enabled
-// if nothing happens, the original source is returned
-func (raw rawCopyCmdArgs) stripTrailingWildcardOnRemoteSource(location common.Location) (result string, stripTopDir bool, err error) {
-	result = raw.src
-	resourceURL, err := url.Parse(result)
-	gURLParts := common.NewGenericResourceURLParts(*resourceURL, location)
-
-	if err != nil {
-		err = fmt.Errorf("failed to parse url %s; %w", result, err)
-		return
-	}
-
-	if strings.Contains(gURLParts.GetContainerName(), "*") {
-		// Disallow container name search and object specifics
-		if gURLParts.GetObjectName() != "" {
-			err = errors.New("cannot combine a specific object name with an account-level search")
-			return
-		}
-
-		// Return immediately here because we know this will be safe.
-		return
-	}
-
-	// Trim the trailing /*.
-	if strings.HasSuffix(resourceURL.RawPath, "/*") {
-		resourceURL.RawPath = strings.TrimSuffix(resourceURL.RawPath, "/*")
-		resourceURL.Path = strings.TrimSuffix(resourceURL.Path, "/*")
-		stripTopDir = true
-	}
-
-	// Ensure there aren't any extra *s floating around.
-	if strings.Contains(resourceURL.RawPath, "*") {
-		err = errors.New("cannot use wildcards in the path section of the URL except in trailing \"/*\". If you wish to use * in your URL, manually encode it to %2A")
-		return
-	}
-
-	result = resourceURL.String()
-
-	return
-}
-
 func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
 	cooked := CookedCopyCmdArgs{
 		jobID: azcopyCurrentJobID,
@@ -298,7 +243,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
 
 	// Check if source has a trailing wildcard on a URL
 	if fromTo.From().IsRemote() {
-		tempSrc, cooked.StripTopDir, err = raw.stripTrailingWildcardOnRemoteSource(fromTo.From())
+		tempSrc, cooked.StripTopDir, err = stripTrailingWildcardOnRemoteSource(raw.src, fromTo.From())
 
 		if err != nil {
 			return cooked, err
@@ -410,7 +355,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
 	}
 
 	// warn on exclude unsupported wildcards here. Include have to be later, to cover list-of-files
-	raw.warnIfHasWildcard(excludeWarningOncer, "exclude-path", raw.excludePath)
+	warnIfHasWildcard(excludeWarningOncer, "exclude-path", raw.excludePath)
 
 	// unbuffered so this reads as we need it to rather than all at once in bulk
 	listChan := make(chan string)
@@ -433,7 +378,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
 		addToChannel := func(v string, paramName string) {
 			// empty strings should be ignored, otherwise the source root itself is selected
 			if len(v) > 0 {
-				raw.warnIfHasWildcard(includeWarningOncer, paramName, v)
+				warnIfHasWildcard(includeWarningOncer, paramName, v)
 				listChan <- v
 			}
 		}
@@ -477,7 +422,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
 		}
 
 		// This occurs much earlier than the other include or exclude filters. It would be preferable to move them closer later on in the refactor.
-		includePathList := raw.parsePatterns(raw.includePath)
+		includePathList := parsePatterns(raw.includePath)
 
 		for _, v := range includePathList {
 			addToChannel(v, "include-path")
@@ -868,19 +813,19 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
 	}
 
 	// parse the filter patterns
-	cooked.IncludePatterns = raw.parsePatterns(raw.include)
-	cooked.ExcludePatterns = raw.parsePatterns(raw.exclude)
-	cooked.ExcludePathPatterns = raw.parsePatterns(raw.excludePath)
-	cooked.excludeContainer = raw.parsePatterns(raw.excludeContainer)
+	cooked.IncludePatterns = parsePatterns(raw.include)
+	cooked.ExcludePatterns = parsePatterns(raw.exclude)
+	cooked.ExcludePathPatterns = parsePatterns(raw.excludePath)
+	cooked.excludeContainer = parsePatterns(raw.excludeContainer)
 
 	if (raw.includeFileAttributes != "" || raw.excludeFileAttributes != "") && fromTo.From() != common.ELocation.Local() {
 		return cooked, errors.New("cannot check file attributes on remote objects")
 	}
-	cooked.IncludeFileAttributes = raw.parsePatterns(raw.includeFileAttributes)
-	cooked.ExcludeFileAttributes = raw.parsePatterns(raw.excludeFileAttributes)
+	cooked.IncludeFileAttributes = parsePatterns(raw.includeFileAttributes)
+	cooked.ExcludeFileAttributes = parsePatterns(raw.excludeFileAttributes)
 
-	cooked.includeRegex = raw.parsePatterns(raw.includeRegex)
-	cooked.excludeRegex = raw.parsePatterns(raw.excludeRegex)
+	cooked.includeRegex = parsePatterns(raw.includeRegex)
+	cooked.excludeRegex = parsePatterns(raw.excludeRegex)
 
 	cooked.dryrunMode = raw.dryrun
 
@@ -903,16 +848,6 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
 var excludeWarningOncer = &sync.Once{}
 var includeWarningOncer = &sync.Once{}
 
-func (raw *rawCopyCmdArgs) warnIfHasWildcard(oncer *sync.Once, paramName string, value string) {
-	if strings.Contains(value, "*") || strings.Contains(value, "?") {
-		oncer.Do(func() {
-			glcm.Warn(fmt.Sprintf("*** Warning *** The %s parameter does not support wildcards. The wildcard "+
-				"character provided will be interpreted literally and will not have any wildcard effect. To use wildcards "+
-				"(in filenames only, not paths) use include-pattern or exclude-pattern", paramName))
-		})
-	}
-}
-
 // When other commands use the copy command arguments to cook cook, set the blobType to None and validation option
 // else parsing the arguments will fail.
 func (raw *rawCopyCmdArgs) setMandatoryDefaults() {

cmd/copyUtil.go

@@ -21,11 +21,13 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"net/url"
 	"os"
 	"strings"
+	"sync"
 
 	"github.com/Azure/azure-storage-azcopy/v10/common"
 )
@@ -34,6 +36,71 @@ const (
 	NumOfFilesPerDispatchJobPart = 10000
 )
 
+func parsePatterns(pattern string) (cookedPatterns []string) {
+	cookedPatterns = make([]string, 0)
+	rawPatterns := strings.Split(pattern, ";")
+	for _, pattern := range rawPatterns {
+
+		// skip the empty patterns
+		if len(pattern) != 0 {
+			cookedPatterns = append(cookedPatterns, pattern)
+		}
+	}
+
+	return
+}
+
+// returns result of stripping and if striptopdir is enabled
+// if nothing happens, the original source is returned
+func stripTrailingWildcardOnRemoteSource(source string, location common.Location) (result string, stripTopDir bool, err error) {
+	result = source
+	resourceURL, err := url.Parse(result)
+	gURLParts := common.NewGenericResourceURLParts(*resourceURL, location)
+
+	if err != nil {
+		err = fmt.Errorf("failed to parse url %s; %w", result, err)
+		return
+	}
+
+	if strings.Contains(gURLParts.GetContainerName(), "*") {
+		// Disallow container name search and object specifics
+		if gURLParts.GetObjectName() != "" {
+			err = errors.New("cannot combine a specific object name with an account-level search")
+			return
+		}
+
+		// Return immediately here because we know this will be safe.
+		return
+	}
+
+	// Trim the trailing /*.
+	if strings.HasSuffix(resourceURL.RawPath, "/*") {
+		resourceURL.RawPath = strings.TrimSuffix(resourceURL.RawPath, "/*")
+		resourceURL.Path = strings.TrimSuffix(resourceURL.Path, "/*")
+		stripTopDir = true
+	}
+
+	// Ensure there aren't any extra *s floating around.
+	if strings.Contains(resourceURL.RawPath, "*") {
+		err = errors.New("cannot use wildcards in the path section of the URL except in trailing \"/*\". If you wish to use * in your URL, manually encode it to %2A")
+		return
+	}
+
+	result = resourceURL.String()
+
+	return
+}
+
+func warnIfHasWildcard(oncer *sync.Once, paramName string, value string) {
+	if strings.Contains(value, "*") || strings.Contains(value, "?") {
+		oncer.Do(func() {
+			glcm.Warn(fmt.Sprintf("*** Warning *** The %s parameter does not support wildcards. The wildcard "+
+				"character provided will be interpreted literally and will not have any wildcard effect. To use wildcards "+
+				"(in filenames only, not paths) use include-pattern or exclude-pattern", paramName))
+		})
+	}
+}
+
 type copyHandlerUtil struct{}
 
 // TODO: Need be replaced with anonymous embedded field technique.

cmd/sync.go

@@ -102,22 +102,8 @@ type rawSyncCmdArgs struct {
 	deleteDestinationFileIfNecessary bool
 }
 
-func (raw *rawSyncCmdArgs) parsePatterns(pattern string) (cookedPatterns []string) {
-	cookedPatterns = make([]string, 0)
-	rawPatterns := strings.Split(pattern, ";")
-	for _, pattern := range rawPatterns {
-
-		// skip the empty patterns
-		if len(pattern) != 0 {
-			cookedPatterns = append(cookedPatterns, pattern)
-		}
-	}
-
-	return
-}
-
 // it is assume that the given url has the SAS stripped, and safe to print
-func (raw *rawSyncCmdArgs) validateURLIsNotServiceLevel(url string, location common.Location) error {
+func validateURLIsNotServiceLevel(url string, location common.Location) error {
 	srcLevel, err := DetermineLocationLevel(url, location, true)
 	if err != nil {
 		return err
@@ -213,15 +199,15 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) {
 
 	// we do not support service level sync yet
 	if cooked.fromTo.From().IsRemote() {
-		err = raw.validateURLIsNotServiceLevel(cooked.source.Value, cooked.fromTo.From())
+		err = validateURLIsNotServiceLevel(cooked.source.Value, cooked.fromTo.From())
 		if err != nil {
 			return cooked, err
 		}
 	}
 
 	// we do not support service level sync yet
 	if cooked.fromTo.To().IsRemote() {
-		err = raw.validateURLIsNotServiceLevel(cooked.destination.Value, cooked.fromTo.To())
+		err = validateURLIsNotServiceLevel(cooked.destination.Value, cooked.fromTo.To())
 		if err != nil {
 			return cooked, err
 		}
@@ -265,13 +251,13 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) {
 	}
 
 	// parse the filter patterns
-	cooked.includePatterns = raw.parsePatterns(raw.include)
-	cooked.excludePatterns = raw.parsePatterns(raw.exclude)
-	cooked.excludePaths = raw.parsePatterns(raw.excludePath)
+	cooked.includePatterns = parsePatterns(raw.include)
+	cooked.excludePatterns = parsePatterns(raw.exclude)
+	cooked.excludePaths = parsePatterns(raw.excludePath)
 
 	// parse the attribute filter patterns
-	cooked.includeFileAttributes = raw.parsePatterns(raw.includeFileAttributes)
-	cooked.excludeFileAttributes = raw.parsePatterns(raw.excludeFileAttributes)
+	cooked.includeFileAttributes = parsePatterns(raw.includeFileAttributes)
+	cooked.excludeFileAttributes = parsePatterns(raw.excludeFileAttributes)
 
 	cooked.preserveSMBInfo = raw.preserveSMBInfo && areBothLocationsSMBAware(cooked.fromTo)
 
@@ -367,8 +353,8 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) {
 
 	cooked.mirrorMode = raw.mirrorMode
 
-	cooked.includeRegex = raw.parsePatterns(raw.includeRegex)
-	cooked.excludeRegex = raw.parsePatterns(raw.excludeRegex)
+	cooked.includeRegex = parsePatterns(raw.includeRegex)
+	cooked.excludeRegex = parsePatterns(raw.excludeRegex)
 
 	cooked.dryrunMode = raw.dryrun
 

cmd/zt_generic_filter_test.go

@@ -38,8 +38,7 @@ var _ = chk.Suite(&genericFilterSuite{})
 func TestIncludeFilter(t *testing.T) {
 	a := assert.New(t)
 	// set up the filters
-	raw := rawSyncCmdArgs{}
-	includePatternList := raw.parsePatterns("*.pdf;*.jpeg;exactName")
+	includePatternList := parsePatterns("*.pdf;*.jpeg;exactName")
 	includeFilter := buildIncludeFilters(includePatternList)[0]
 
 	// test the positive cases
@@ -60,8 +59,7 @@ func TestIncludeFilter(t *testing.T) {
 func TestExcludeFilter(t *testing.T) {
 	a := assert.New(t)
 	// set up the filters
-	raw := rawSyncCmdArgs{}
-	excludePatternList := raw.parsePatterns("*.pdf;*.jpeg;exactName")
+	excludePatternList := parsePatterns("*.pdf;*.jpeg;exactName")
 	excludeFilterList := buildExcludeFilters(excludePatternList, false)
 
 	// test the positive cases
@@ -187,4 +185,4 @@ func findAmbiguousTime() (string, time.Time, time.Time, error) {
 	}
 
 	return "", time.Time{}, time.Time{}, noAmbiguousHourError
-}
+}