Skip to content

Commit a334832

Browse files
adreed-msftnakulkar-msft
authored andcommitted
Fix wildcard handling when a file of the same name as the wildcard input exists (#2062)
* Fix stgexp bug * Fix test compilation
1 parent a3749ee commit a334832

10 files changed

+96
-20
lines changed

cmd/copyEnumeratorInit.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,8 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
7171
traverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &srcCredInfo,
7272
cca.SymlinkHandling, cca.ListOfFilesChannel, cca.Recursive, getRemoteProperties,
7373
cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs,
74-
cca.S2sPreserveBlobTags, common.ESyncHashType.None(), cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */)
74+
cca.S2sPreserveBlobTags, common.ESyncHashType.None(), cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(),
75+
cca.CpkOptions, nil /* errorChannel */, cca.StripTopDir)
7576

7677
if err != nil {
7778
return nil, err
@@ -338,7 +339,8 @@ func (cca *CookedCopyCmdArgs) isDestDirectory(dst common.ResourceString, ctx *co
338339

339340
rt, err := InitResourceTraverser(dst, cca.FromTo.To(), ctx, &dstCredInfo, common.ESymlinkHandlingType.Skip(),
340341
nil, false, false, false, common.EPermanentDeleteOption.None(),
341-
func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), cca.preservePermissions, pipeline.LogNone, cca.CpkOptions, nil /* errorChannel */)
342+
func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), cca.preservePermissions, pipeline.LogNone,
343+
cca.CpkOptions, nil /* errorChannel */, cca.StripTopDir)
342344

343345
if err != nil {
344346
return false

cmd/list.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,8 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) {
224224

225225
traverser, err := InitResourceTraverser(source, cooked.location, &ctx, &credentialInfo, common.ESymlinkHandlingType.Skip(), nil,
226226
true, false, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {},
227-
nil, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), pipeline.LogNone, common.CpkOptions{}, nil /* errorChannel */)
227+
nil, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(),
228+
pipeline.LogNone, common.CpkOptions{}, nil /* errorChannel */, false)
228229

229230
if err != nil {
230231
return fmt.Errorf("failed to initialize traverser: %s", err.Error())

cmd/removeEnumerator.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ func newRemoveEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator, er
5151
sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo,
5252
common.ESymlinkHandlingType.Skip(), cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs,
5353
cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false,
54-
common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */)
54+
common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity.ToPipelineLogLevel(),
55+
cca.CpkOptions, nil /* errorChannel */, cca.StripTopDir)
5556

5657
// report failure to create traverser
5758
if err != nil {

cmd/setPropertiesEnumerator.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ func setPropertiesEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator
5151
common.ESymlinkHandlingType.Preserve(), // preserve because we want to index all blobs, including symlink blobs
5252
cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs,
5353
cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false,
54-
common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */)
54+
common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity.ToPipelineLogLevel(),
55+
cca.CpkOptions, nil /* errorChannel */, cca.StripTopDir)
5556

5657
// report failure to create traverser
5758
if err != nil {

cmd/syncEnumerator.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,8 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s
6565
if entityType == common.EEntityType.File() {
6666
atomic.AddUint64(&cca.atomicSourceFilesScanned, 1)
6767
}
68-
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */)
68+
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(),
69+
cca.cpkOptions, nil /* errorChannel */, false)
6970

7071
if err != nil {
7172
return nil, err
@@ -86,7 +87,8 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s
8687
if entityType == common.EEntityType.File() {
8788
atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1)
8889
}
89-
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */)
90+
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(),
91+
cca.cpkOptions, nil /* errorChannel */, false)
9092
if err != nil {
9193
return nil, err
9294
}

cmd/zc_enumerator.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,7 @@ type enumerationCounterFunc func(entityType common.EntityType)
332332
func InitResourceTraverser(resource common.ResourceString, location common.Location, ctx *context.Context,
333333
credential *common.CredentialInfo, symlinkHandling common.SymlinkHandlingType, listOfFilesChannel chan string, recursive, getProperties,
334334
includeDirectoryStubs bool, permanentDeleteOption common.PermanentDeleteOption, incrementEnumerationCounter enumerationCounterFunc, listOfVersionIds chan string,
335-
s2sPreserveBlobTags bool, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption, logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo) (ResourceTraverser, error) {
335+
s2sPreserveBlobTags bool, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption, logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo, stripTopDir bool) (ResourceTraverser, error) {
336336
var output ResourceTraverser
337337
var p *pipeline.Pipeline
338338

@@ -389,7 +389,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat
389389
_, err := common.OSStat(resource.ValueLocal())
390390

391391
// If wildcard is present and this isn't an existing file/folder, glob and feed the globbed list into a list enum.
392-
if strings.Contains(resource.ValueLocal(), "*") && err != nil {
392+
if strings.Contains(resource.ValueLocal(), "*") && (stripTopDir || err != nil) {
393393
basePath := getPathBeforeFirstWildcard(resource.ValueLocal())
394394
matches, err := filepath.Glob(resource.ValueLocal())
395395

@@ -411,9 +411,9 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat
411411
globChan, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions, syncHashType, preservePermissions)
412412
} else {
413413
if ctx != nil {
414-
output = newLocalTraverser(*ctx, resource.ValueLocal(), recursive, symlinkHandling, syncHashType, incrementEnumerationCounter, errorChannel)
414+
output = newLocalTraverser(*ctx, resource.ValueLocal(), recursive, stripTopDir, symlinkHandling, syncHashType, incrementEnumerationCounter, errorChannel)
415415
} else {
416-
output = newLocalTraverser(context.TODO(), resource.ValueLocal(), recursive, symlinkHandling, syncHashType, incrementEnumerationCounter, errorChannel)
416+
output = newLocalTraverser(context.TODO(), resource.ValueLocal(), recursive, stripTopDir, symlinkHandling, syncHashType, incrementEnumerationCounter, errorChannel)
417417
}
418418
}
419419
case common.ELocation.Benchmark():

cmd/zc_traverser_list.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ func newListTraverser(parent common.ResourceString, parentType common.Location,
108108
// Construct a traverser that goes through the child
109109
traverser, err := InitResourceTraverser(source, parentType, ctx, credential, handleSymlinks,
110110
nil, recursive, getProperties, includeDirectoryStubs, common.EPermanentDeleteOption.None(), incrementEnumerationCounter,
111-
nil, s2sPreserveBlobTags, syncHashType, preservePermissions, logLevel, cpkOptions, nil /* errorChannel */)
111+
nil, s2sPreserveBlobTags, syncHashType, preservePermissions, logLevel, cpkOptions, nil /* errorChannel */, false)
112112
if err != nil {
113113
return nil, err
114114
}

cmd/zc_traverser_local.go

+9-2
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ const MAX_SYMLINKS_TO_FOLLOW = 40
4545
type localTraverser struct {
4646
fullPath string
4747
recursive bool
48+
stripTopDir bool
4849
symlinkHandling common.SymlinkHandlingType
4950
appCtx context.Context
5051
// a generic function to notify that a new stored object has been enumerated
@@ -71,6 +72,10 @@ func (t *localTraverser) IsDirectory(bool) (bool, error) {
7172
}
7273

7374
func (t *localTraverser) getInfoIfSingleFile() (os.FileInfo, bool, error) {
75+
if t.stripTopDir {
76+
return nil, false, nil // StripTopDir can NEVER be a single file. If a user wants to target a single file, they must escape the *.
77+
}
78+
7479
fileInfo, err := common.OSStat(t.fullPath)
7580

7681
if err != nil {
@@ -793,15 +798,17 @@ func (t *localTraverser) Traverse(preprocessor objectMorpher, processor objectPr
793798
return finalizer(err)
794799
}
795800

796-
func newLocalTraverser(ctx context.Context, fullPath string, recursive bool, symlinkHandling common.SymlinkHandlingType, syncHashType common.SyncHashType, incrementEnumerationCounter enumerationCounterFunc, errorChannel chan ErrorFileInfo) *localTraverser {
801+
func newLocalTraverser(ctx context.Context, fullPath string, recursive bool, stripTopDir bool, symlinkHandling common.SymlinkHandlingType, syncHashType common.SyncHashType, incrementEnumerationCounter enumerationCounterFunc, errorChannel chan ErrorFileInfo) *localTraverser {
797802
traverser := localTraverser{
798803
fullPath: cleanLocalPath(fullPath),
799804
recursive: recursive,
800805
symlinkHandling: symlinkHandling,
801806
appCtx: ctx,
802807
incrementEnumerationCounter: incrementEnumerationCounter,
803808
errorChannel: errorChannel,
804-
targetHashType: syncHashType}
809+
targetHashType: syncHashType,
810+
stripTopDir: stripTopDir,
811+
}
805812
return &traverser
806813
}
807814

cmd/zt_generic_service_traverser_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ func (s *genericTraverserSuite) TestBlobFSServiceTraverserWithManyObjects(c *chk
5858
scenarioHelper{}.generateLocalFilesFromList(c, dstDirName, objectList)
5959

6060
// Create a local traversal
61-
localTraverser := newLocalTraverser(context.TODO(), dstDirName, true, common.ESymlinkHandlingType.Follow(), common.ESyncHashType.None(), func(common.EntityType) {}, nil)
61+
localTraverser := newLocalTraverser(context.TODO(), dstDirName, true, false, common.ESymlinkHandlingType.Follow(), common.ESyncHashType.None(), func(common.EntityType) {}, nil)
6262

6363
// Invoke the traversal with an indexer so the results are indexed for easy validation
6464
localIndexer := newObjectIndexer()
@@ -174,7 +174,7 @@ func (s *genericTraverserSuite) TestServiceTraverserWithManyObjects(c *chk.C) {
174174
scenarioHelper{}.generateLocalFilesFromList(c, dstDirName, objectList)
175175

176176
// Create a local traversal
177-
localTraverser := newLocalTraverser(context.TODO(), dstDirName, true, common.ESymlinkHandlingType.Follow(), common.ESyncHashType.None(), func(common.EntityType) {}, nil)
177+
localTraverser := newLocalTraverser(context.TODO(), dstDirName, true, false, common.ESymlinkHandlingType.Follow(), common.ESyncHashType.None(), func(common.EntityType) {}, nil)
178178

179179
// Invoke the traversal with an indexer so the results are indexed for easy validation
180180
localIndexer := newObjectIndexer()
@@ -358,7 +358,7 @@ func (s *genericTraverserSuite) TestServiceTraverserWithWildcards(c *chk.C) {
358358
scenarioHelper{}.generateLocalFilesFromList(c, dstDirName, objectList)
359359

360360
// Create a local traversal
361-
localTraverser := newLocalTraverser(context.TODO(), dstDirName, true, common.ESymlinkHandlingType.Follow(), common.ESyncHashType.None(), func(common.EntityType) {}, nil)
361+
localTraverser := newLocalTraverser(context.TODO(), dstDirName, true, false, common.ESymlinkHandlingType.Follow(), common.ESyncHashType.None(), func(common.EntityType) {}, nil)
362362

363363
// Invoke the traversal with an indexer so the results are indexed for easy validation
364364
localIndexer := newObjectIndexer()

cmd/zt_generic_traverser_test.go

+65-3
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@ package cmd
2222

2323
import (
2424
"context"
25+
"github.com/Azure/azure-pipeline-go/pipeline"
2526
"io"
2627
"os"
2728
"path/filepath"
29+
"runtime"
2830
"strings"
2931
"time"
3032

@@ -55,6 +57,66 @@ func trySymlink(src, dst string, c *chk.C) {
5557
}
5658
}
5759

60+
func (s *genericTraverserSuite) TestLocalWildcardOverlap(c *chk.C) {
61+
if runtime.GOOS == "windows" {
62+
c.Skip("invalid filename used")
63+
return
64+
}
65+
66+
/*
67+
Wildcard support is not actually a part of the local traverser, believe it or not.
68+
It's instead implemented in InitResourceTraverser as a short-circuit to a list traverser
69+
utilizing the filepath.Glob function, which then initializes local traversers to achieve the same effect.
70+
*/
71+
tmpDir := scenarioHelper{}.generateLocalDirectory(c)
72+
defer func(path string) { _ = os.RemoveAll(path) }(tmpDir)
73+
74+
scenarioHelper{}.generateLocalFilesFromList(c, tmpDir, []string{
75+
"test.txt",
76+
"tes*t.txt",
77+
"foobarbaz/test.txt",
78+
})
79+
80+
resource, err := SplitResourceString(filepath.Join(tmpDir, "tes*t.txt"), common.ELocation.Local())
81+
c.Assert(err, chk.IsNil)
82+
83+
traverser, err := InitResourceTraverser(
84+
resource,
85+
common.ELocation.Local(),
86+
nil,
87+
nil,
88+
common.ESymlinkHandlingType.Follow(),
89+
nil,
90+
true,
91+
false,
92+
false,
93+
common.EPermanentDeleteOption.None(),
94+
nil,
95+
nil,
96+
false,
97+
common.ESyncHashType.None(),
98+
common.EPreservePermissionsOption.None(),
99+
pipeline.LogInfo,
100+
common.CpkOptions{},
101+
nil,
102+
true,
103+
)
104+
c.Assert(err, chk.IsNil)
105+
106+
seenFiles := make(map[string]bool)
107+
108+
err = traverser.Traverse(nil, func(storedObject StoredObject) error {
109+
seenFiles[storedObject.relativePath] = true
110+
return nil
111+
}, []ObjectFilter{})
112+
c.Assert(err, chk.IsNil)
113+
114+
c.Assert(seenFiles, chk.DeepEquals, map[string]bool{
115+
"test.txt": true,
116+
"tes*t.txt": true,
117+
})
118+
}
119+
58120
// GetProperties tests.
59121
// GetProperties does not exist on Blob, as the properties come in the list call.
60122
// While BlobFS could get properties in the future, it's currently disabled as BFS source S2S isn't set up right now, and likely won't be.
@@ -483,7 +545,7 @@ func (s *genericTraverserSuite) TestTraverserWithSingleObject(c *chk.C) {
483545
scenarioHelper{}.generateLocalFilesFromList(c, dstDirName, blobList)
484546

485547
// construct a local traverser
486-
localTraverser := newLocalTraverser(context.TODO(), filepath.Join(dstDirName, dstFileName), false, common.ESymlinkHandlingType.Skip(), common.ESyncHashType.None(), func(common.EntityType) {}, nil)
548+
localTraverser := newLocalTraverser(context.TODO(), filepath.Join(dstDirName, dstFileName), false, false, common.ESymlinkHandlingType.Follow(), common.ESyncHashType.None(), func(common.EntityType) {}, nil)
487549

488550
// invoke the local traversal with a dummy processor
489551
localDummyProcessor := dummyProcessor{}
@@ -643,7 +705,7 @@ func (s *genericTraverserSuite) TestTraverserContainerAndLocalDirectory(c *chk.C
643705
// test two scenarios, either recursive or not
644706
for _, isRecursiveOn := range []bool{true, false} {
645707
// construct a local traverser
646-
localTraverser := newLocalTraverser(context.TODO(), dstDirName, isRecursiveOn, common.ESymlinkHandlingType.Skip(), common.ESyncHashType.None(), func(common.EntityType) {}, nil)
708+
localTraverser := newLocalTraverser(context.TODO(), dstDirName, isRecursiveOn, false, common.ESymlinkHandlingType.Follow(), common.ESyncHashType.None(), func(common.EntityType) {}, nil)
647709

648710
// invoke the local traversal with an indexer
649711
// so that the results are indexed for easy validation
@@ -804,7 +866,7 @@ func (s *genericTraverserSuite) TestTraverserWithVirtualAndLocalDirectory(c *chk
804866
// test two scenarios, either recursive or not
805867
for _, isRecursiveOn := range []bool{true, false} {
806868
// construct a local traverser
807-
localTraverser := newLocalTraverser(context.TODO(), filepath.Join(dstDirName, virDirName), isRecursiveOn, common.ESymlinkHandlingType.Skip(), common.ESyncHashType.None(), func(common.EntityType) {}, nil)
869+
localTraverser := newLocalTraverser(context.TODO(), filepath.Join(dstDirName, virDirName), isRecursiveOn, false, common.ESymlinkHandlingType.Follow(), common.ESyncHashType.None(), func(common.EntityType) {}, nil)
808870

809871
// invoke the local traversal with an indexer
810872
// so that the results are indexed for easy validation

0 commit comments

Comments
 (0)