Skip to content

Commit dfd8554

Browse files
Merge branch 'main' into adreed/e2e-test-inherit-auth
2 parents c427506 + 55cef84 commit dfd8554

File tree

6 files changed

+95
-111
lines changed

6 files changed

+95
-111
lines changed

.github/CODEOWNERS

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
* @gapra-msft @adreed-msft @dphulkar-msft @vibhansa-msft @seanmcc-msft @souravgupta-msft @tanyasethi-msft
1+
* @gapra-msft @adreed-msft @dphulkar-msft @vibhansa-msft @seanmcc-msft @souravgupta-msft @tanyasethi-msft @wonwuakpa-msft

.github/workflows/update-wiki-docs.yml

+2-4
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,8 @@
11
name: Update wiki docs
22

33
on:
4-
push:
5-
branches: [ "main" ]
6-
pull_request:
7-
branches: [ "main" ]
4+
release:
5+
types: [published]
86

97
concurrency:
108
group: wiki

cmd/copy.go

+12-77
Original file line numberDiff line numberDiff line change
@@ -189,20 +189,6 @@ type rawCopyCmdArgs struct {
189189
deleteDestinationFileIfNecessary bool
190190
}
191191

192-
func (raw *rawCopyCmdArgs) parsePatterns(pattern string) (cookedPatterns []string) {
193-
cookedPatterns = make([]string, 0)
194-
rawPatterns := strings.Split(pattern, ";")
195-
for _, pattern := range rawPatterns {
196-
197-
// skip the empty patterns
198-
if len(pattern) != 0 {
199-
cookedPatterns = append(cookedPatterns, pattern)
200-
}
201-
}
202-
203-
return
204-
}
205-
206192
// blocSizeInBytes converts a FLOATING POINT number of MiB, to a number of bytes
207193
// A non-nil error is returned if the conversion is not possible to do accurately (e.g. it comes out of a fractional number of bytes)
208194
// The purpose of using floating point is to allow specialist users (e.g. those who want small block sizes to tune their read IOPS)
@@ -226,47 +212,6 @@ func blockSizeInBytes(rawBlockSizeInMiB float64) (int64, error) {
226212
return int64(math.Round(rawSizeInBytes)), nil
227213
}
228214

229-
// returns result of stripping and if striptopdir is enabled
230-
// if nothing happens, the original source is returned
231-
func (raw rawCopyCmdArgs) stripTrailingWildcardOnRemoteSource(location common.Location) (result string, stripTopDir bool, err error) {
232-
result = raw.src
233-
resourceURL, err := url.Parse(result)
234-
gURLParts := common.NewGenericResourceURLParts(*resourceURL, location)
235-
236-
if err != nil {
237-
err = fmt.Errorf("failed to parse url %s; %w", result, err)
238-
return
239-
}
240-
241-
if strings.Contains(gURLParts.GetContainerName(), "*") {
242-
// Disallow container name search and object specifics
243-
if gURLParts.GetObjectName() != "" {
244-
err = errors.New("cannot combine a specific object name with an account-level search")
245-
return
246-
}
247-
248-
// Return immediately here because we know this will be safe.
249-
return
250-
}
251-
252-
// Trim the trailing /*.
253-
if strings.HasSuffix(resourceURL.RawPath, "/*") {
254-
resourceURL.RawPath = strings.TrimSuffix(resourceURL.RawPath, "/*")
255-
resourceURL.Path = strings.TrimSuffix(resourceURL.Path, "/*")
256-
stripTopDir = true
257-
}
258-
259-
// Ensure there aren't any extra *s floating around.
260-
if strings.Contains(resourceURL.RawPath, "*") {
261-
err = errors.New("cannot use wildcards in the path section of the URL except in trailing \"/*\". If you wish to use * in your URL, manually encode it to %2A")
262-
return
263-
}
264-
265-
result = resourceURL.String()
266-
267-
return
268-
}
269-
270215
func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
271216
cooked := CookedCopyCmdArgs{
272217
jobID: azcopyCurrentJobID,
@@ -298,7 +243,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
298243

299244
// Check if source has a trailing wildcard on a URL
300245
if fromTo.From().IsRemote() {
301-
tempSrc, cooked.StripTopDir, err = raw.stripTrailingWildcardOnRemoteSource(fromTo.From())
246+
tempSrc, cooked.StripTopDir, err = stripTrailingWildcardOnRemoteSource(raw.src, fromTo.From())
302247

303248
if err != nil {
304249
return cooked, err
@@ -410,7 +355,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
410355
}
411356

412357
// warn on exclude unsupported wildcards here. Include have to be later, to cover list-of-files
413-
raw.warnIfHasWildcard(excludeWarningOncer, "exclude-path", raw.excludePath)
358+
warnIfHasWildcard(excludeWarningOncer, "exclude-path", raw.excludePath)
414359

415360
// unbuffered so this reads as we need it to rather than all at once in bulk
416361
listChan := make(chan string)
@@ -433,7 +378,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
433378
addToChannel := func(v string, paramName string) {
434379
// empty strings should be ignored, otherwise the source root itself is selected
435380
if len(v) > 0 {
436-
raw.warnIfHasWildcard(includeWarningOncer, paramName, v)
381+
warnIfHasWildcard(includeWarningOncer, paramName, v)
437382
listChan <- v
438383
}
439384
}
@@ -477,7 +422,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
477422
}
478423

479424
// This occurs much earlier than the other include or exclude filters. It would be preferable to move them closer later on in the refactor.
480-
includePathList := raw.parsePatterns(raw.includePath)
425+
includePathList := parsePatterns(raw.includePath)
481426

482427
for _, v := range includePathList {
483428
addToChannel(v, "include-path")
@@ -868,19 +813,19 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
868813
}
869814

870815
// parse the filter patterns
871-
cooked.IncludePatterns = raw.parsePatterns(raw.include)
872-
cooked.ExcludePatterns = raw.parsePatterns(raw.exclude)
873-
cooked.ExcludePathPatterns = raw.parsePatterns(raw.excludePath)
874-
cooked.excludeContainer = raw.parsePatterns(raw.excludeContainer)
816+
cooked.IncludePatterns = parsePatterns(raw.include)
817+
cooked.ExcludePatterns = parsePatterns(raw.exclude)
818+
cooked.ExcludePathPatterns = parsePatterns(raw.excludePath)
819+
cooked.excludeContainer = parsePatterns(raw.excludeContainer)
875820

876821
if (raw.includeFileAttributes != "" || raw.excludeFileAttributes != "") && fromTo.From() != common.ELocation.Local() {
877822
return cooked, errors.New("cannot check file attributes on remote objects")
878823
}
879-
cooked.IncludeFileAttributes = raw.parsePatterns(raw.includeFileAttributes)
880-
cooked.ExcludeFileAttributes = raw.parsePatterns(raw.excludeFileAttributes)
824+
cooked.IncludeFileAttributes = parsePatterns(raw.includeFileAttributes)
825+
cooked.ExcludeFileAttributes = parsePatterns(raw.excludeFileAttributes)
881826

882-
cooked.includeRegex = raw.parsePatterns(raw.includeRegex)
883-
cooked.excludeRegex = raw.parsePatterns(raw.excludeRegex)
827+
cooked.includeRegex = parsePatterns(raw.includeRegex)
828+
cooked.excludeRegex = parsePatterns(raw.excludeRegex)
884829

885830
cooked.dryrunMode = raw.dryrun
886831

@@ -903,16 +848,6 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
903848
var excludeWarningOncer = &sync.Once{}
904849
var includeWarningOncer = &sync.Once{}
905850

906-
func (raw *rawCopyCmdArgs) warnIfHasWildcard(oncer *sync.Once, paramName string, value string) {
907-
if strings.Contains(value, "*") || strings.Contains(value, "?") {
908-
oncer.Do(func() {
909-
glcm.Warn(fmt.Sprintf("*** Warning *** The %s parameter does not support wildcards. The wildcard "+
910-
"character provided will be interpreted literally and will not have any wildcard effect. To use wildcards "+
911-
"(in filenames only, not paths) use include-pattern or exclude-pattern", paramName))
912-
})
913-
}
914-
}
915-
916851
// When other commands use the copy command arguments to cook cook, set the blobType to None and validation option
917852
// else parsing the arguments will fail.
918853
func (raw *rawCopyCmdArgs) setMandatoryDefaults() {

cmd/copyUtil.go

+67
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,13 @@
2121
package cmd
2222

2323
import (
24+
"errors"
2425
"fmt"
2526
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
2627
"net/url"
2728
"os"
2829
"strings"
30+
"sync"
2931

3032
"github.com/Azure/azure-storage-azcopy/v10/common"
3133
)
@@ -34,6 +36,71 @@ const (
3436
NumOfFilesPerDispatchJobPart = 10000
3537
)
3638

39+
func parsePatterns(pattern string) (cookedPatterns []string) {
40+
cookedPatterns = make([]string, 0)
41+
rawPatterns := strings.Split(pattern, ";")
42+
for _, pattern := range rawPatterns {
43+
44+
// skip the empty patterns
45+
if len(pattern) != 0 {
46+
cookedPatterns = append(cookedPatterns, pattern)
47+
}
48+
}
49+
50+
return
51+
}
52+
53+
// returns result of stripping and if striptopdir is enabled
54+
// if nothing happens, the original source is returned
55+
func stripTrailingWildcardOnRemoteSource(source string, location common.Location) (result string, stripTopDir bool, err error) {
56+
result = source
57+
resourceURL, err := url.Parse(result)
58+
gURLParts := common.NewGenericResourceURLParts(*resourceURL, location)
59+
60+
if err != nil {
61+
err = fmt.Errorf("failed to parse url %s; %w", result, err)
62+
return
63+
}
64+
65+
if strings.Contains(gURLParts.GetContainerName(), "*") {
66+
// Disallow container name search and object specifics
67+
if gURLParts.GetObjectName() != "" {
68+
err = errors.New("cannot combine a specific object name with an account-level search")
69+
return
70+
}
71+
72+
// Return immediately here because we know this will be safe.
73+
return
74+
}
75+
76+
// Trim the trailing /*.
77+
if strings.HasSuffix(resourceURL.RawPath, "/*") {
78+
resourceURL.RawPath = strings.TrimSuffix(resourceURL.RawPath, "/*")
79+
resourceURL.Path = strings.TrimSuffix(resourceURL.Path, "/*")
80+
stripTopDir = true
81+
}
82+
83+
// Ensure there aren't any extra *s floating around.
84+
if strings.Contains(resourceURL.RawPath, "*") {
85+
err = errors.New("cannot use wildcards in the path section of the URL except in trailing \"/*\". If you wish to use * in your URL, manually encode it to %2A")
86+
return
87+
}
88+
89+
result = resourceURL.String()
90+
91+
return
92+
}
93+
94+
func warnIfHasWildcard(oncer *sync.Once, paramName string, value string) {
95+
if strings.Contains(value, "*") || strings.Contains(value, "?") {
96+
oncer.Do(func() {
97+
glcm.Warn(fmt.Sprintf("*** Warning *** The %s parameter does not support wildcards. The wildcard "+
98+
"character provided will be interpreted literally and will not have any wildcard effect. To use wildcards "+
99+
"(in filenames only, not paths) use include-pattern or exclude-pattern", paramName))
100+
})
101+
}
102+
}
103+
37104
type copyHandlerUtil struct{}
38105

39106
// TODO: Need be replaced with anonymous embedded field technique.

cmd/sync.go

+10-24
Original file line numberDiff line numberDiff line change
@@ -102,22 +102,8 @@ type rawSyncCmdArgs struct {
102102
deleteDestinationFileIfNecessary bool
103103
}
104104

105-
func (raw *rawSyncCmdArgs) parsePatterns(pattern string) (cookedPatterns []string) {
106-
cookedPatterns = make([]string, 0)
107-
rawPatterns := strings.Split(pattern, ";")
108-
for _, pattern := range rawPatterns {
109-
110-
// skip the empty patterns
111-
if len(pattern) != 0 {
112-
cookedPatterns = append(cookedPatterns, pattern)
113-
}
114-
}
115-
116-
return
117-
}
118-
119105
// it is assume that the given url has the SAS stripped, and safe to print
120-
func (raw *rawSyncCmdArgs) validateURLIsNotServiceLevel(url string, location common.Location) error {
106+
func validateURLIsNotServiceLevel(url string, location common.Location) error {
121107
srcLevel, err := DetermineLocationLevel(url, location, true)
122108
if err != nil {
123109
return err
@@ -213,15 +199,15 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) {
213199

214200
// we do not support service level sync yet
215201
if cooked.fromTo.From().IsRemote() {
216-
err = raw.validateURLIsNotServiceLevel(cooked.source.Value, cooked.fromTo.From())
202+
err = validateURLIsNotServiceLevel(cooked.source.Value, cooked.fromTo.From())
217203
if err != nil {
218204
return cooked, err
219205
}
220206
}
221207

222208
// we do not support service level sync yet
223209
if cooked.fromTo.To().IsRemote() {
224-
err = raw.validateURLIsNotServiceLevel(cooked.destination.Value, cooked.fromTo.To())
210+
err = validateURLIsNotServiceLevel(cooked.destination.Value, cooked.fromTo.To())
225211
if err != nil {
226212
return cooked, err
227213
}
@@ -265,13 +251,13 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) {
265251
}
266252

267253
// parse the filter patterns
268-
cooked.includePatterns = raw.parsePatterns(raw.include)
269-
cooked.excludePatterns = raw.parsePatterns(raw.exclude)
270-
cooked.excludePaths = raw.parsePatterns(raw.excludePath)
254+
cooked.includePatterns = parsePatterns(raw.include)
255+
cooked.excludePatterns = parsePatterns(raw.exclude)
256+
cooked.excludePaths = parsePatterns(raw.excludePath)
271257

272258
// parse the attribute filter patterns
273-
cooked.includeFileAttributes = raw.parsePatterns(raw.includeFileAttributes)
274-
cooked.excludeFileAttributes = raw.parsePatterns(raw.excludeFileAttributes)
259+
cooked.includeFileAttributes = parsePatterns(raw.includeFileAttributes)
260+
cooked.excludeFileAttributes = parsePatterns(raw.excludeFileAttributes)
275261

276262
cooked.preserveSMBInfo = raw.preserveSMBInfo && areBothLocationsSMBAware(cooked.fromTo)
277263

@@ -367,8 +353,8 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) {
367353

368354
cooked.mirrorMode = raw.mirrorMode
369355

370-
cooked.includeRegex = raw.parsePatterns(raw.includeRegex)
371-
cooked.excludeRegex = raw.parsePatterns(raw.excludeRegex)
356+
cooked.includeRegex = parsePatterns(raw.includeRegex)
357+
cooked.excludeRegex = parsePatterns(raw.excludeRegex)
372358

373359
cooked.dryrunMode = raw.dryrun
374360

cmd/zt_generic_filter_test.go

+3-5
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,7 @@ var _ = chk.Suite(&genericFilterSuite{})
3838
func TestIncludeFilter(t *testing.T) {
3939
a := assert.New(t)
4040
// set up the filters
41-
raw := rawSyncCmdArgs{}
42-
includePatternList := raw.parsePatterns("*.pdf;*.jpeg;exactName")
41+
includePatternList := parsePatterns("*.pdf;*.jpeg;exactName")
4342
includeFilter := buildIncludeFilters(includePatternList)[0]
4443

4544
// test the positive cases
@@ -60,8 +59,7 @@ func TestIncludeFilter(t *testing.T) {
6059
func TestExcludeFilter(t *testing.T) {
6160
a := assert.New(t)
6261
// set up the filters
63-
raw := rawSyncCmdArgs{}
64-
excludePatternList := raw.parsePatterns("*.pdf;*.jpeg;exactName")
62+
excludePatternList := parsePatterns("*.pdf;*.jpeg;exactName")
6563
excludeFilterList := buildExcludeFilters(excludePatternList, false)
6664

6765
// test the positive cases
@@ -187,4 +185,4 @@ func findAmbiguousTime() (string, time.Time, time.Time, error) {
187185
}
188186

189187
return "", time.Time{}, time.Time{}, noAmbiguousHourError
190-
}
188+
}

0 commit comments

Comments
 (0)