From f7377a4d4dca18e2e7ac33f853e015033a02b561 Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:17:24 -0500 Subject: [PATCH 01/12] fixing list version property and added test --- cmd/zc_traverser_blob.go | 2 + cmd/zt_list_test.go | 65 +++++++++++++++++++++++++++++ cmd/zt_scenario_helpers_for_test.go | 33 +++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 cmd/zt_list_test.go diff --git a/cmd/zc_traverser_blob.go b/cmd/zc_traverser_blob.go index acfbd032c..1612ea3a2 100644 --- a/cmd/zc_traverser_blob.go +++ b/cmd/zc_traverser_blob.go @@ -478,6 +478,8 @@ func (t *blobTraverser) createStoredObjectForBlob(preprocessor objectMorpher, bl object.blobSnapshotID = common.IffNotNil(blobInfo.Snapshot, "") } else if t.includeDeleted && t.includeVersion && blobInfo.VersionID != nil { object.blobVersionID = common.IffNotNil(blobInfo.VersionID, "") + } else if blobInfo.VersionID != nil { + object.blobVersionID = common.IffNotNil(blobInfo.VersionID, "") } return object } diff --git a/cmd/zt_list_test.go b/cmd/zt_list_test.go new file mode 100644 index 000000000..5a0649765 --- /dev/null +++ b/cmd/zt_list_test.go @@ -0,0 +1,65 @@ +package cmd + +import ( + "github.com/Azure/azure-storage-azcopy/v10/common" + "github.com/stretchr/testify/assert" + "strings" + "testing" +) + +func TestListVersions(t *testing.T) { + a := assert.New(t) + bsc := getSecondaryBlobServiceClient() + // set up the container with single blob with 2 versions + containerClient, containerName := createNewContainer(a, bsc) + defer deleteContainer(a, containerClient) + + blobsToInclude := []string{"AzURE2021.jpeg", "sub1/dir2/HELLO-4.txt", "sub1/test/testing.txt"} + scenarioHelper{}.generateBlobsFromList(a, containerClient, blobsToInclude, blockBlobDefaultData) + a.NotNil(containerClient) + + // get dictionary/map of blob: version id + versions := make(map[string]string) + for _, blob := range blobsToInclude { + props, err := containerClient.NewBlockBlobClient(blob).GetProperties(ctx, nil) + a.NoError(err) + + versions[blob] = *props.VersionID + } + + // confirm that container has 3 blobs + pager := containerClient.NewListBlobsFlatPager(nil) + list, err := pager.NextPage(ctx) + a.NoError(err) + a.NotNil(list.Segment.BlobItems) + a.Equal(3, len(list.Segment.BlobItems)) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + mockedLcm := mockedLifecycleManager{infoLog: make(chan string, 50)} + mockedLcm.SetOutputFormat(common.EOutputFormat.Text()) // text format + glcm = &mockedLcm + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getSecondaryRawContainerURLWithSAS(a, containerName) + raw := getDefaultListRawInput(rawContainerURLWithSAS.String()) + raw.Properties = "VersionId" + + runListAndVerify(a, raw, func(err error) { + a.Nil(err) + + // validate that the no transfers were scheduled + a.Nil(mockedRPC.transfers) + + // check if info logs contain the correct version id for each blob + msg := mockedLcm.GatherAllLogs(mockedLcm.infoLog) + for i, m := range msg { + a.True(strings.Contains(m, blobsToInclude[i])) + a.True(strings.Contains(m, "VersionId: "+versions[blobsToInclude[i]])) + } + }) + +} diff --git a/cmd/zt_scenario_helpers_for_test.go b/cmd/zt_scenario_helpers_for_test.go index 4b834f818..5e3cbad08 100644 --- a/cmd/zt_scenario_helpers_for_test.go +++ b/cmd/zt_scenario_helpers_for_test.go @@ -599,6 +599,17 @@ func (scenarioHelper) getRawContainerURLWithSAS(a *assert.Assertions, containerN return parsedURL } +func (scenarioHelper) getSecondaryRawContainerURLWithSAS(a *assert.Assertions, containerName string) *url.URL { + accountName, accountKey := getSecondaryAccountAndKey() + credential, err := blob.NewSharedKeyCredential(accountName, accountKey) + a.Nil(err) + cc := getContainerClientWithSAS(a, credential, containerName) + + u := cc.URL() + parsedURL, err := url.Parse(u) + return parsedURL +} + func (scenarioHelper) getContainerClientWithSAS(a *assert.Assertions, containerName string) *container.Client { accountName, accountKey := getAccountAndKey() credential, err := blob.NewSharedKeyCredential(accountName, accountKey) @@ -858,6 +869,17 @@ func runCopyAndVerify(a *assert.Assertions, raw rawCopyCmdArgs, verifier func(er verifier(err) } +func runListAndVerify(a *assert.Assertions, raw rawListCmdArgs, verifier func(err error)) { + // the simulated user input should parse properly + cooked, err := raw.cook() + a.NoError(err) + + err = cooked.HandleListContainerCommand() + + // the err is passed to verified, which knows whether it is expected or not + verifier(err) +} + func validateUploadTransfersAreScheduled(a *assert.Assertions, sourcePrefix string, destinationPrefix string, expectedTransfers []string, mockedRPC interceptor) { validateCopyTransfersAreScheduled(a, false, true, sourcePrefix, destinationPrefix, expectedTransfers, mockedRPC) } @@ -989,6 +1011,17 @@ func getDefaultRemoveRawInput(src string) rawCopyCmdArgs { } } +func getDefaultListRawInput(src string) rawListCmdArgs { + return rawListCmdArgs{ + sourcePath: src, + Properties: "", + MachineReadable: false, + RunningTally: false, + MegaUnits: false, + trailingDot: "", + } +} + func getDefaultSetPropertiesRawInput(src string, params transferParams) rawCopyCmdArgs { fromTo := common.EFromTo.BlobNone() srcURL, _ := url.Parse(src) From 87a6f0be8fc36a054e142d37a436d5bb45684195 Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Wed, 24 Jan 2024 13:06:05 -0500 Subject: [PATCH 02/12] only add vid when intended, add test for multi-versions --- cmd/copyEnumeratorInit.go | 4 +- cmd/copyEnumeratorInit_test.go | 12 +++--- cmd/list.go | 12 +++++- cmd/removeEnumerator.go | 2 +- cmd/setPropertiesEnumerator.go | 2 +- cmd/syncEnumerator.go | 4 +- cmd/zc_enumerator.go | 6 +-- cmd/zc_traverser_blob.go | 13 ++++--- cmd/zc_traverser_blob_account.go | 2 +- cmd/zc_traverser_list.go | 2 +- cmd/zt_generic_traverser_test.go | 11 +++--- cmd/zt_list_test.go | 65 ++++++++++++++++++++++++++++++++ cmd/zt_traverser_blob_test.go | 20 +++++----- 13 files changed, 116 insertions(+), 39 deletions(-) diff --git a/cmd/copyEnumeratorInit.go b/cmd/copyEnumeratorInit.go index 8ffb6be1f..0495b6407 100755 --- a/cmd/copyEnumeratorInit.go +++ b/cmd/copyEnumeratorInit.go @@ -72,7 +72,7 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde jobPartOrder.S2SPreserveBlobTags = cca.S2sPreserveBlobTags dest := cca.FromTo.To() - traverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &srcCredInfo, cca.SymlinkHandling, cca.ListOfFilesChannel, cca.Recursive, getRemoteProperties, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, cca.S2sPreserveBlobTags, common.ESyncHashType.None(), cca.preservePermissions, azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, &dest, cca.excludeContainer) + traverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &srcCredInfo, cca.SymlinkHandling, cca.ListOfFilesChannel, cca.Recursive, getRemoteProperties, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, cca.S2sPreserveBlobTags, common.ESyncHashType.None(), cca.preservePermissions, azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, &dest, cca.excludeContainer, false) if err != nil { return nil, err @@ -343,7 +343,7 @@ func (cca *CookedCopyCmdArgs) isDestDirectory(dst common.ResourceString, ctx *co return false } - rt, err := InitResourceTraverser(dst, cca.FromTo.To(), ctx, &dstCredInfo, common.ESymlinkHandlingType.Skip(), nil, false, false, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), cca.preservePermissions, common.LogNone, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil, cca.excludeContainer) + rt, err := InitResourceTraverser(dst, cca.FromTo.To(), ctx, &dstCredInfo, common.ESymlinkHandlingType.Skip(), nil, false, false, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), cca.preservePermissions, common.LogNone, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil, cca.excludeContainer, false) if err != nil { return false diff --git a/cmd/copyEnumeratorInit_test.go b/cmd/copyEnumeratorInit_test.go index 87c6e306a..fa7c95f6d 100644 --- a/cmd/copyEnumeratorInit_test.go +++ b/cmd/copyEnumeratorInit_test.go @@ -46,7 +46,7 @@ func TestValidateSourceDirThatExists(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, dirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) // dir but recursive flag not set - fail cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} @@ -76,7 +76,7 @@ func TestValidateSourceDirDoesNotExist(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, dirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) // dir but recursive flag not set - fail cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} @@ -107,7 +107,7 @@ func TestValidateSourceFileExists(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, fileName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} err := cca.validateSourceDir(blobTraverser) @@ -131,7 +131,7 @@ func TestValidateSourceFileDoesNotExist(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, fileName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} err := cca.validateSourceDir(blobTraverser) @@ -155,11 +155,11 @@ func TestValidateSourceWithWildCard(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, dirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) // dir but recursive flag not set - fail cca := CookedCopyCmdArgs{StripTopDir: true, Recursive: false} err := cca.validateSourceDir(blobTraverser) a.Nil(err) a.False(cca.IsSourceDir) -} \ No newline at end of file +} diff --git a/cmd/list.go b/cmd/list.go index 80e609c71..b71819912 100755 --- a/cmd/list.go +++ b/cmd/list.go @@ -61,6 +61,16 @@ const ( archiveStatus validProperty = "ArchiveStatus" ) +// containsProperty checks if the property array contains a valid property +func containsProperty(properties []validProperty, prop validProperty) bool { + for _, item := range properties { + if item == prop { + return true + } + } + return false +} + // validProperties returns an array of possible values for the validProperty const type. func validProperties() []validProperty { return []validProperty{lastModifiedTime, versionId, blobType, blobAccessTier, @@ -237,7 +247,7 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { } } - traverser, err := InitResourceTraverser(source, cooked.location, &ctx, &credentialInfo, common.ESymlinkHandlingType.Skip(), nil, true, true, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, nil, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), common.LogNone, common.CpkOptions{}, nil, false, cooked.trailingDot, nil, nil) + traverser, err := InitResourceTraverser(source, cooked.location, &ctx, &credentialInfo, common.ESymlinkHandlingType.Skip(), nil, true, true, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, nil, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), common.LogNone, common.CpkOptions{}, nil, false, cooked.trailingDot, nil, nil, containsProperty(cooked.properties, versionId)) if err != nil { return fmt.Errorf("failed to initialize traverser: %s", err.Error()) diff --git a/cmd/removeEnumerator.go b/cmd/removeEnumerator.go index 6e830f1bc..01692bde0 100755 --- a/cmd/removeEnumerator.go +++ b/cmd/removeEnumerator.go @@ -50,7 +50,7 @@ func newRemoveEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator, er ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) // Include-path is handled by ListOfFilesChannel. - sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo, common.ESymlinkHandlingType.Skip(), cca.ListOfFilesChannel, cca.Recursive, true, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil, cca.excludeContainer) + sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo, common.ESymlinkHandlingType.Skip(), cca.ListOfFilesChannel, cca.Recursive, true, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil, cca.excludeContainer, false) // report failure to create traverser if err != nil { diff --git a/cmd/setPropertiesEnumerator.go b/cmd/setPropertiesEnumerator.go index f61bcef90..f2ebce28d 100755 --- a/cmd/setPropertiesEnumerator.go +++ b/cmd/setPropertiesEnumerator.go @@ -46,7 +46,7 @@ func setPropertiesEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator } // Include-path is handled by ListOfFilesChannel. - sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo, common.ESymlinkHandlingType.Preserve(), cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil, cca.excludeContainer) + sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo, common.ESymlinkHandlingType.Preserve(), cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil, cca.excludeContainer, false) // report failure to create traverser if err != nil { diff --git a/cmd/syncEnumerator.go b/cmd/syncEnumerator.go index e0d52205a..0fc6d4750 100644 --- a/cmd/syncEnumerator.go +++ b/cmd/syncEnumerator.go @@ -64,7 +64,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s if entityType == common.EEntityType.File() { atomic.AddUint64(&cca.atomicSourceFilesScanned, 1) } - }, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity, cca.cpkOptions, nil, false, cca.trailingDot, &dest, nil) + }, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity, cca.cpkOptions, nil, false, cca.trailingDot, &dest, nil, false) if err != nil { return nil, err @@ -85,7 +85,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s if entityType == common.EEntityType.File() { atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1) } - }, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity, cca.cpkOptions, nil, false, cca.trailingDot, nil, nil) + }, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity, cca.cpkOptions, nil, false, cca.trailingDot, nil, nil, false) if err != nil { return nil, err } diff --git a/cmd/zc_enumerator.go b/cmd/zc_enumerator.go index 39740306f..1557593ae 100644 --- a/cmd/zc_enumerator.go +++ b/cmd/zc_enumerator.go @@ -334,7 +334,7 @@ type enumerationCounterFunc func(entityType common.EntityType) // errorOnDirWOutRecursive is used by copy. // If errorChannel is non-nil, all errors encountered during enumeration will be conveyed through this channel. // To avoid slowdowns, use a buffered channel of enough capacity. -func InitResourceTraverser(resource common.ResourceString, location common.Location, ctx *context.Context, credential *common.CredentialInfo, symlinkHandling common.SymlinkHandlingType, listOfFilesChannel chan string, recursive, getProperties, includeDirectoryStubs bool, permanentDeleteOption common.PermanentDeleteOption, incrementEnumerationCounter enumerationCounterFunc, listOfVersionIds chan string, s2sPreserveBlobTags bool, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption, logLevel common.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo, stripTopDir bool, trailingDot common.TrailingDotOption, destination *common.Location, excludeContainerNames []string) (ResourceTraverser, error) { +func InitResourceTraverser(resource common.ResourceString, location common.Location, ctx *context.Context, credential *common.CredentialInfo, symlinkHandling common.SymlinkHandlingType, listOfFilesChannel chan string, recursive, getProperties, includeDirectoryStubs bool, permanentDeleteOption common.PermanentDeleteOption, incrementEnumerationCounter enumerationCounterFunc, listOfVersionIds chan string, s2sPreserveBlobTags bool, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption, logLevel common.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo, stripTopDir bool, trailingDot common.TrailingDotOption, destination *common.Location, excludeContainerNames []string, includeVersionsList bool) (ResourceTraverser, error) { var output ResourceTraverser var includeDeleted bool @@ -457,7 +457,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat } else if listOfVersionIds != nil { output = newBlobVersionsTraverser(r, bsc, *ctx, includeDirectoryStubs, incrementEnumerationCounter, listOfVersionIds, cpkOptions) } else { - output = newBlobTraverser(r, bsc, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion, preservePermissions, false) + output = newBlobTraverser(r, bsc, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion, preservePermissions, false, includeVersionsList) } case common.ELocation.File(): // TODO (last service migration) : Remove dependency on URLs. @@ -544,7 +544,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat } else if listOfVersionIds != nil { output = newBlobVersionsTraverser(r, bsc, *ctx, includeDirectoryStubs, incrementEnumerationCounter, listOfVersionIds, cpkOptions) } else { - output = newBlobTraverser(r, bsc, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion, preservePermissions, true) + output = newBlobTraverser(r, bsc, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion, preservePermissions, true, false) } case common.ELocation.S3(): resourceURL, err := resource.FullURL() diff --git a/cmd/zc_traverser_blob.go b/cmd/zc_traverser_blob.go index 1612ea3a2..5974a3bd8 100644 --- a/cmd/zc_traverser_blob.go +++ b/cmd/zc_traverser_blob.go @@ -68,6 +68,8 @@ type blobTraverser struct { includeVersion bool + includeVersionList bool + isDFS bool } @@ -319,7 +321,7 @@ func (t *blobTraverser) parallelList(containerClient *container.Client, containe pager := containerClient.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{ Prefix: ¤tDirPath, - Include: container.ListBlobsInclude{Metadata: true, Tags: t.s2sPreserveSourceTags, Deleted: t.includeDeleted, Snapshots: t.includeSnapshot, Versions: t.includeVersion}, + Include: container.ListBlobsInclude{Metadata: true, Tags: t.s2sPreserveSourceTags, Deleted: t.includeDeleted, Snapshots: t.includeSnapshot, Versions: t.includeVersion || t.includeVersionList}, }) var marker *string for pager.More() { @@ -476,9 +478,7 @@ func (t *blobTraverser) createStoredObjectForBlob(preprocessor objectMorpher, bl object.blobDeleted = common.IffNotNil(blobInfo.Deleted, false) if t.includeDeleted && t.includeSnapshot { object.blobSnapshotID = common.IffNotNil(blobInfo.Snapshot, "") - } else if t.includeDeleted && t.includeVersion && blobInfo.VersionID != nil { - object.blobVersionID = common.IffNotNil(blobInfo.VersionID, "") - } else if blobInfo.VersionID != nil { + } else if (t.includeVersionList || (t.includeDeleted && t.includeVersion)) && blobInfo.VersionID != nil { object.blobVersionID = common.IffNotNil(blobInfo.VersionID, "") } return object @@ -497,7 +497,7 @@ func (t *blobTraverser) serialList(containerClient *container.Client, containerN prefix := searchPrefix + extraSearchPrefix pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ Prefix: &prefix, - Include: container.ListBlobsInclude{Metadata: true, Tags: t.s2sPreserveSourceTags, Deleted: t.includeDeleted, Snapshots: t.includeSnapshot, Versions: t.includeVersion}, + Include: container.ListBlobsInclude{Metadata: true, Tags: t.s2sPreserveSourceTags, Deleted: t.includeDeleted, Snapshots: t.includeSnapshot, Versions: t.includeVersion || t.includeVersionList}, }) for pager.More() { resp, err := pager.NextPage(t.ctx) @@ -543,7 +543,7 @@ func (t *blobTraverser) serialList(containerClient *container.Client, containerN return nil } -func newBlobTraverser(rawURL string, serviceClient *service.Client, ctx context.Context, recursive, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, includeDeleted, includeSnapshot, includeVersion bool, preservePermissions common.PreservePermissionsOption, isDFS bool) (t *blobTraverser) { +func newBlobTraverser(rawURL string, serviceClient *service.Client, ctx context.Context, recursive, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, includeDeleted, includeSnapshot, includeVersion bool, preservePermissions common.PreservePermissionsOption, isDFS bool, includeVersionList bool) (t *blobTraverser) { t = &blobTraverser{ rawURL: rawURL, serviceClient: serviceClient, @@ -557,6 +557,7 @@ func newBlobTraverser(rawURL string, serviceClient *service.Client, ctx context. includeDeleted: includeDeleted, includeSnapshot: includeSnapshot, includeVersion: includeVersion, + includeVersionList: includeVersionList, preservePermissions: preservePermissions, isDFS: isDFS, } diff --git a/cmd/zc_traverser_blob_account.go b/cmd/zc_traverser_blob_account.go index 458c7c009..30705bb2a 100644 --- a/cmd/zc_traverser_blob_account.go +++ b/cmd/zc_traverser_blob_account.go @@ -120,7 +120,7 @@ func (t *blobAccountTraverser) Traverse(preprocessor objectMorpher, processor ob for _, v := range cList { containerURL := t.serviceClient.NewContainerClient(v).URL() - containerTraverser := newBlobTraverser(containerURL, t.serviceClient, t.ctx, true, t.includeDirectoryStubs, t.incrementEnumerationCounter, t.s2sPreserveSourceTags, t.cpkOptions, false, false, false, t.preservePermissions, t.isDFS) + containerTraverser := newBlobTraverser(containerURL, t.serviceClient, t.ctx, true, t.includeDirectoryStubs, t.incrementEnumerationCounter, t.s2sPreserveSourceTags, t.cpkOptions, false, false, false, t.preservePermissions, t.isDFS, false) preprocessorForThisChild := preprocessor.FollowedBy(newContainerDecorator(v)) diff --git a/cmd/zc_traverser_list.go b/cmd/zc_traverser_list.go index fa96f898d..540d5f26c 100755 --- a/cmd/zc_traverser_list.go +++ b/cmd/zc_traverser_list.go @@ -107,7 +107,7 @@ func newListTraverser(parent common.ResourceString, parentType common.Location, } // Construct a traverser that goes through the child - traverser, err := InitResourceTraverser(source, parentType, ctx, credential, handleSymlinks, nil, recursive, getProperties, includeDirectoryStubs, common.EPermanentDeleteOption.None(), incrementEnumerationCounter, nil, s2sPreserveBlobTags, syncHashType, preservePermissions, logLevel, cpkOptions, nil, false, trailingDot, destination, nil) + traverser, err := InitResourceTraverser(source, parentType, ctx, credential, handleSymlinks, nil, recursive, getProperties, includeDirectoryStubs, common.EPermanentDeleteOption.None(), incrementEnumerationCounter, nil, s2sPreserveBlobTags, syncHashType, preservePermissions, logLevel, cpkOptions, nil, false, trailingDot, destination, nil, false) if err != nil { return nil, err } diff --git a/cmd/zt_generic_traverser_test.go b/cmd/zt_generic_traverser_test.go index 7f976a038..e491153ad 100644 --- a/cmd/zt_generic_traverser_test.go +++ b/cmd/zt_generic_traverser_test.go @@ -104,6 +104,7 @@ func TestLocalWildcardOverlap(t *testing.T) { common.ETrailingDotOption.Enable(), nil, nil, + false, ) a.Nil(err) @@ -572,7 +573,7 @@ func TestTraverserWithSingleObject(t *testing.T) { ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, blobList[0]).URL() blobServiceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, blobServiceClientWithSAS, ctx, false, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, blobServiceClientWithSAS, ctx, false, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) // invoke the blob traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -714,7 +715,7 @@ func TestTraverserContainerAndLocalDirectory(t *testing.T) { ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) rawContainerURLWithSAS := scenarioHelper{}.getContainerClientWithSAS(a, containerName).URL() blobServiceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawContainerURLWithSAS) - blobTraverser := newBlobTraverser(rawContainerURLWithSAS, blobServiceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + blobTraverser := newBlobTraverser(rawContainerURLWithSAS, blobServiceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) // invoke the local traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -863,7 +864,7 @@ func TestTraverserWithVirtualAndLocalDirectory(t *testing.T) { ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) rawVirDirURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, virDirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawVirDirURLWithSAS) - blobTraverser := newBlobTraverser(rawVirDirURLWithSAS, serviceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + blobTraverser := newBlobTraverser(rawVirDirURLWithSAS, serviceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) // invoke the local traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -960,10 +961,10 @@ func TestSerialAndParallelBlobTraverser(t *testing.T) { ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) rawVirDirURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, virDirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawVirDirURLWithSAS) - parallelBlobTraverser := newBlobTraverser(rawVirDirURLWithSAS, serviceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + parallelBlobTraverser := newBlobTraverser(rawVirDirURLWithSAS, serviceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) // construct a serial blob traverser - serialBlobTraverser := newBlobTraverser(rawVirDirURLWithSAS, serviceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + serialBlobTraverser := newBlobTraverser(rawVirDirURLWithSAS, serviceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) serialBlobTraverser.parallelListing = false // invoke the parallel traversal with a dummy processor diff --git a/cmd/zt_list_test.go b/cmd/zt_list_test.go index 5a0649765..36fdd2e97 100644 --- a/cmd/zt_list_test.go +++ b/cmd/zt_list_test.go @@ -1,6 +1,8 @@ package cmd import ( + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/Azure/azure-storage-azcopy/v10/common" "github.com/stretchr/testify/assert" "strings" @@ -63,3 +65,66 @@ func TestListVersions(t *testing.T) { }) } + +func TestListVersionsMultiVersions(t *testing.T) { + a := assert.New(t) + bsc := getSecondaryBlobServiceClient() + // set up the container with single blob with 2 versions + containerClient, containerName := createNewContainer(a, bsc) + defer deleteContainer(a, containerClient) + + blobsToInclude := []string{"foo.txt", "sub1/dir2/bar.txt", "sub1/test/baz.txt"} + scenarioHelper{}.generateBlobsFromList(a, containerClient, blobsToInclude, blockBlobDefaultData) + a.NotNil(containerClient) + + // make first blob have another version + bbClient := containerClient.NewBlockBlobClient(blobsToInclude[0]) + uploadResp, err := bbClient.Upload(ctx, streaming.NopCloser(strings.NewReader("Random random")), nil) + a.NoError(err) + a.NotNil(uploadResp.VersionID) + + // confirm that container has 3 blobs + pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ + Include: container.ListBlobsInclude{Versions: true}, + }) + list, err := pager.NextPage(ctx) + a.NoError(err) + a.NotNil(list.Segment.BlobItems) + a.Equal(4, len(list.Segment.BlobItems)) + + var blobs []string + var versions []string + for _, item := range list.Segment.BlobItems { + blobs = append(blobs, *item.Name) + versions = append(versions, *item.VersionID) + } + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + mockedLcm := mockedLifecycleManager{infoLog: make(chan string, 50)} + mockedLcm.SetOutputFormat(common.EOutputFormat.Text()) // text format + glcm = &mockedLcm + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getSecondaryRawContainerURLWithSAS(a, containerName) + raw := getDefaultListRawInput(rawContainerURLWithSAS.String()) + raw.Properties = "VersionId" + + runListAndVerify(a, raw, func(err error) { + a.Nil(err) + + // validate that the no transfers were scheduled + a.Nil(mockedRPC.transfers) + + // check if info logs contain the correct version id for each blob + msg := mockedLcm.GatherAllLogs(mockedLcm.infoLog) + for i, m := range msg { + a.True(strings.Contains(m, blobs[i])) + a.True(strings.Contains(m, versions[i])) + } + }) + +} diff --git a/cmd/zt_traverser_blob_test.go b/cmd/zt_traverser_blob_test.go index a663418e5..a02a66b2f 100644 --- a/cmd/zt_traverser_blob_test.go +++ b/cmd/zt_traverser_blob_test.go @@ -49,7 +49,7 @@ func TestIsSourceDirWithStub(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, dirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) isDir, err := blobTraverser.IsDirectory(true) a.True(isDir) @@ -71,7 +71,7 @@ func TestIsSourceDirWithNoStub(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, dirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) isDir, err := blobTraverser.IsDirectory(true) a.True(isDir) @@ -93,7 +93,7 @@ func TestIsDestDirWithBlobEP(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, dirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) isDir, err := blobTraverser.IsDirectory(false) a.True(isDir) @@ -103,7 +103,7 @@ func TestIsDestDirWithBlobEP(t *testing.T) { dirName = "dest_file" // List rawBlobURLWithSAS = scenarioHelper{}.getBlobClientWithSAS(a, containerName, dirName).URL() - blobTraverser = newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + blobTraverser = newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) isDir, err = blobTraverser.IsDirectory(false) a.False(isDir) @@ -129,9 +129,9 @@ func TestIsDestDirWithDFSEP(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, fileSystemName, parentDirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), true) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), true, false) - // a directory with name parentDirName exists on target. So irrespective of + // a directory with name parentDirName exists on target. So irrespective of // isSource, IsDirectory() should return true. isDir, err := blobTraverser.IsDirectory(true) a.True(isDir) @@ -146,7 +146,7 @@ func TestIsDestDirWithDFSEP(t *testing.T) { // With a directory that does not exist, without path separator. parentDirName = "dirDoesNotExist" rawBlobURLWithSAS = scenarioHelper{}.getBlobClientWithSAS(a, fileSystemName, parentDirName).URL() - blobTraverser = newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), true) + blobTraverser = newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), true, false) // The directory does not exist, so IsDirectory() // should return false, in all cases @@ -163,7 +163,7 @@ func TestIsDestDirWithDFSEP(t *testing.T) { // With a directory that does not exist, with path separator parentDirNameWithSeparator := "dirDoesNotExist" + common.OS_PATH_SEPARATOR rawBlobURLWithSAS = scenarioHelper{}.getBlobClientWithSAS(a, fileSystemName, parentDirNameWithSeparator).URL() - blobTraverser = newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), true) + blobTraverser = newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), true, false) // The directory does not exist, but with a path separator // we should identify it as a directory. @@ -194,7 +194,7 @@ func TestIsSourceFileExists(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, fileName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) isDir, err := blobTraverser.IsDirectory(true) a.False(isDir) @@ -216,7 +216,7 @@ func TestIsSourceFileDoesNotExist(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, fileName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) isDir, err := blobTraverser.IsDirectory(true) a.False(isDir) From 2efa87357498b4956d6d59595b67a75ff789215e Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Wed, 24 Jan 2024 16:31:13 -0500 Subject: [PATCH 03/12] addressing feedback --- cmd/copyEnumeratorInit_test.go | 10 +++++----- cmd/zc_enumerator.go | 8 ++++++-- cmd/zc_traverser_blob.go | 11 ++++------- cmd/zc_traverser_blob_account.go | 2 +- cmd/zt_generic_traverser_test.go | 10 +++++----- cmd/zt_traverser_blob_test.go | 18 +++++++++--------- 6 files changed, 30 insertions(+), 29 deletions(-) diff --git a/cmd/copyEnumeratorInit_test.go b/cmd/copyEnumeratorInit_test.go index fa7c95f6d..ed41a32b9 100644 --- a/cmd/copyEnumeratorInit_test.go +++ b/cmd/copyEnumeratorInit_test.go @@ -46,7 +46,7 @@ func TestValidateSourceDirThatExists(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, dirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) // dir but recursive flag not set - fail cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} @@ -76,7 +76,7 @@ func TestValidateSourceDirDoesNotExist(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, dirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) // dir but recursive flag not set - fail cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} @@ -107,7 +107,7 @@ func TestValidateSourceFileExists(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, fileName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} err := cca.validateSourceDir(blobTraverser) @@ -131,7 +131,7 @@ func TestValidateSourceFileDoesNotExist(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, fileName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} err := cca.validateSourceDir(blobTraverser) @@ -155,7 +155,7 @@ func TestValidateSourceWithWildCard(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, dirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) // dir but recursive flag not set - fail cca := CookedCopyCmdArgs{StripTopDir: true, Recursive: false} diff --git a/cmd/zc_enumerator.go b/cmd/zc_enumerator.go index 1557593ae..1a75cab3f 100644 --- a/cmd/zc_enumerator.go +++ b/cmd/zc_enumerator.go @@ -353,6 +353,10 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat includeVersion = true } + // print out version id when using azcopy list + if includeVersionsList { + includeVersion = true + } // Clean up the resource if it's a local path if location == common.ELocation.Local() { resource = common.ResourceString{Value: cleanLocalPath(resource.ValueLocal())} @@ -457,7 +461,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat } else if listOfVersionIds != nil { output = newBlobVersionsTraverser(r, bsc, *ctx, includeDirectoryStubs, incrementEnumerationCounter, listOfVersionIds, cpkOptions) } else { - output = newBlobTraverser(r, bsc, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion, preservePermissions, false, includeVersionsList) + output = newBlobTraverser(r, bsc, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion, preservePermissions, false) } case common.ELocation.File(): // TODO (last service migration) : Remove dependency on URLs. @@ -544,7 +548,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat } else if listOfVersionIds != nil { output = newBlobVersionsTraverser(r, bsc, *ctx, includeDirectoryStubs, incrementEnumerationCounter, listOfVersionIds, cpkOptions) } else { - output = newBlobTraverser(r, bsc, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion, preservePermissions, true, false) + output = newBlobTraverser(r, bsc, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion, preservePermissions, true) } case common.ELocation.S3(): resourceURL, err := resource.FullURL() diff --git a/cmd/zc_traverser_blob.go b/cmd/zc_traverser_blob.go index 5974a3bd8..42a43dd46 100644 --- a/cmd/zc_traverser_blob.go +++ b/cmd/zc_traverser_blob.go @@ -68,8 +68,6 @@ type blobTraverser struct { includeVersion bool - includeVersionList bool - isDFS bool } @@ -321,7 +319,7 @@ func (t *blobTraverser) parallelList(containerClient *container.Client, containe pager := containerClient.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{ Prefix: ¤tDirPath, - Include: container.ListBlobsInclude{Metadata: true, Tags: t.s2sPreserveSourceTags, Deleted: t.includeDeleted, Snapshots: t.includeSnapshot, Versions: t.includeVersion || t.includeVersionList}, + Include: container.ListBlobsInclude{Metadata: true, Tags: t.s2sPreserveSourceTags, Deleted: t.includeDeleted, Snapshots: t.includeSnapshot, Versions: t.includeVersion}, }) var marker *string for pager.More() { @@ -478,7 +476,7 @@ func (t *blobTraverser) createStoredObjectForBlob(preprocessor objectMorpher, bl object.blobDeleted = common.IffNotNil(blobInfo.Deleted, false) if t.includeDeleted && t.includeSnapshot { object.blobSnapshotID = common.IffNotNil(blobInfo.Snapshot, "") - } else if (t.includeVersionList || (t.includeDeleted && t.includeVersion)) && blobInfo.VersionID != nil { + } else if t.includeVersion && blobInfo.VersionID != nil { object.blobVersionID = common.IffNotNil(blobInfo.VersionID, "") } return object @@ -497,7 +495,7 @@ func (t *blobTraverser) serialList(containerClient *container.Client, containerN prefix := searchPrefix + extraSearchPrefix pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ Prefix: &prefix, - Include: container.ListBlobsInclude{Metadata: true, Tags: t.s2sPreserveSourceTags, Deleted: t.includeDeleted, Snapshots: t.includeSnapshot, Versions: t.includeVersion || t.includeVersionList}, + Include: container.ListBlobsInclude{Metadata: true, Tags: t.s2sPreserveSourceTags, Deleted: t.includeDeleted, Snapshots: t.includeSnapshot, Versions: t.includeVersion}, }) for pager.More() { resp, err := pager.NextPage(t.ctx) @@ -543,7 +541,7 @@ func (t *blobTraverser) serialList(containerClient *container.Client, containerN return nil } -func newBlobTraverser(rawURL string, serviceClient *service.Client, ctx context.Context, recursive, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, includeDeleted, includeSnapshot, includeVersion bool, preservePermissions common.PreservePermissionsOption, isDFS bool, includeVersionList bool) (t *blobTraverser) { +func newBlobTraverser(rawURL string, serviceClient *service.Client, ctx context.Context, recursive, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, includeDeleted, includeSnapshot, includeVersion bool, preservePermissions common.PreservePermissionsOption, isDFS bool) (t *blobTraverser) { t = &blobTraverser{ rawURL: rawURL, serviceClient: serviceClient, @@ -557,7 +555,6 @@ func newBlobTraverser(rawURL string, serviceClient *service.Client, ctx context. includeDeleted: includeDeleted, includeSnapshot: includeSnapshot, includeVersion: includeVersion, - includeVersionList: includeVersionList, preservePermissions: preservePermissions, isDFS: isDFS, } diff --git a/cmd/zc_traverser_blob_account.go b/cmd/zc_traverser_blob_account.go index 30705bb2a..458c7c009 100644 --- a/cmd/zc_traverser_blob_account.go +++ b/cmd/zc_traverser_blob_account.go @@ -120,7 +120,7 @@ func (t *blobAccountTraverser) Traverse(preprocessor objectMorpher, processor ob for _, v := range cList { containerURL := t.serviceClient.NewContainerClient(v).URL() - containerTraverser := newBlobTraverser(containerURL, t.serviceClient, t.ctx, true, t.includeDirectoryStubs, t.incrementEnumerationCounter, t.s2sPreserveSourceTags, t.cpkOptions, false, false, false, t.preservePermissions, t.isDFS, false) + containerTraverser := newBlobTraverser(containerURL, t.serviceClient, t.ctx, true, t.includeDirectoryStubs, t.incrementEnumerationCounter, t.s2sPreserveSourceTags, t.cpkOptions, false, false, false, t.preservePermissions, t.isDFS) preprocessorForThisChild := preprocessor.FollowedBy(newContainerDecorator(v)) diff --git a/cmd/zt_generic_traverser_test.go b/cmd/zt_generic_traverser_test.go index e491153ad..d7af5cbb8 100644 --- a/cmd/zt_generic_traverser_test.go +++ b/cmd/zt_generic_traverser_test.go @@ -573,7 +573,7 @@ func TestTraverserWithSingleObject(t *testing.T) { ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, blobList[0]).URL() blobServiceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, blobServiceClientWithSAS, ctx, false, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, blobServiceClientWithSAS, ctx, false, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) // invoke the blob traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -715,7 +715,7 @@ func TestTraverserContainerAndLocalDirectory(t *testing.T) { ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) rawContainerURLWithSAS := scenarioHelper{}.getContainerClientWithSAS(a, containerName).URL() blobServiceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawContainerURLWithSAS) - blobTraverser := newBlobTraverser(rawContainerURLWithSAS, blobServiceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + blobTraverser := newBlobTraverser(rawContainerURLWithSAS, blobServiceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) // invoke the local traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -864,7 +864,7 @@ func TestTraverserWithVirtualAndLocalDirectory(t *testing.T) { ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) rawVirDirURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, virDirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawVirDirURLWithSAS) - blobTraverser := newBlobTraverser(rawVirDirURLWithSAS, serviceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + blobTraverser := newBlobTraverser(rawVirDirURLWithSAS, serviceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) // invoke the local traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -961,10 +961,10 @@ func TestSerialAndParallelBlobTraverser(t *testing.T) { ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) rawVirDirURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, virDirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawVirDirURLWithSAS) - parallelBlobTraverser := newBlobTraverser(rawVirDirURLWithSAS, serviceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + parallelBlobTraverser := newBlobTraverser(rawVirDirURLWithSAS, serviceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) // construct a serial blob traverser - serialBlobTraverser := newBlobTraverser(rawVirDirURLWithSAS, serviceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + serialBlobTraverser := newBlobTraverser(rawVirDirURLWithSAS, serviceClientWithSAS, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) serialBlobTraverser.parallelListing = false // invoke the parallel traversal with a dummy processor diff --git a/cmd/zt_traverser_blob_test.go b/cmd/zt_traverser_blob_test.go index a02a66b2f..513cc69e8 100644 --- a/cmd/zt_traverser_blob_test.go +++ b/cmd/zt_traverser_blob_test.go @@ -49,7 +49,7 @@ func TestIsSourceDirWithStub(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, dirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) isDir, err := blobTraverser.IsDirectory(true) a.True(isDir) @@ -71,7 +71,7 @@ func TestIsSourceDirWithNoStub(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, dirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) isDir, err := blobTraverser.IsDirectory(true) a.True(isDir) @@ -93,7 +93,7 @@ func TestIsDestDirWithBlobEP(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, dirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) isDir, err := blobTraverser.IsDirectory(false) a.True(isDir) @@ -103,7 +103,7 @@ func TestIsDestDirWithBlobEP(t *testing.T) { dirName = "dest_file" // List rawBlobURLWithSAS = scenarioHelper{}.getBlobClientWithSAS(a, containerName, dirName).URL() - blobTraverser = newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + blobTraverser = newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) isDir, err = blobTraverser.IsDirectory(false) a.False(isDir) @@ -129,7 +129,7 @@ func TestIsDestDirWithDFSEP(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, fileSystemName, parentDirName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), true, false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), true) // a directory with name parentDirName exists on target. So irrespective of // isSource, IsDirectory() should return true. @@ -146,7 +146,7 @@ func TestIsDestDirWithDFSEP(t *testing.T) { // With a directory that does not exist, without path separator. parentDirName = "dirDoesNotExist" rawBlobURLWithSAS = scenarioHelper{}.getBlobClientWithSAS(a, fileSystemName, parentDirName).URL() - blobTraverser = newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), true, false) + blobTraverser = newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), true) // The directory does not exist, so IsDirectory() // should return false, in all cases @@ -163,7 +163,7 @@ func TestIsDestDirWithDFSEP(t *testing.T) { // With a directory that does not exist, with path separator parentDirNameWithSeparator := "dirDoesNotExist" + common.OS_PATH_SEPARATOR rawBlobURLWithSAS = scenarioHelper{}.getBlobClientWithSAS(a, fileSystemName, parentDirNameWithSeparator).URL() - blobTraverser = newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), true, false) + blobTraverser = newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), true) // The directory does not exist, but with a path separator // we should identify it as a directory. @@ -194,7 +194,7 @@ func TestIsSourceFileExists(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, fileName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) isDir, err := blobTraverser.IsDirectory(true) a.False(isDir) @@ -216,7 +216,7 @@ func TestIsSourceFileDoesNotExist(t *testing.T) { // List rawBlobURLWithSAS := scenarioHelper{}.getBlobClientWithSAS(a, containerName, fileName).URL() serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, rawBlobURLWithSAS) - blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false, false) + blobTraverser := newBlobTraverser(rawBlobURLWithSAS, serviceClientWithSAS, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false) isDir, err := blobTraverser.IsDirectory(true) a.False(isDir) From 8584b9ccd132779a389c9c8c782c46cf0eb3bce5 Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Thu, 25 Jan 2024 15:57:50 -0500 Subject: [PATCH 04/12] handle running tally + updates to test with running tally --- cmd/list.go | 15 +++++++++------ cmd/zt_list_test.go | 26 ++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/cmd/list.go b/cmd/list.go index b71819912..414fd0054 100755 --- a/cmd/list.go +++ b/cmd/list.go @@ -255,15 +255,23 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { var fileCount int64 = 0 var sizeCount int64 = 0 + var objectSummary string processor := func(object StoredObject) error { + if cooked.RunningTally { + if !(strings.Contains(objectSummary, object.name) && containsProperty(cooked.properties, versionId)) { + fileCount++ + sizeCount += object.size + } + } + path := object.relativePath if object.entityType == common.EEntityType.Folder() { path += "/" // TODO: reviewer: same questions as for jobs status: OK to hard code direction of slash? OK to use trailing slash to distinguish dirs from files? } properties := "; " + cooked.processProperties(object) - objectSummary := path + properties + " Content Length: " + objectSummary = path + properties + " Content Length: " if level == level.Service() { objectSummary = object.ContainerName + "/" + objectSummary @@ -275,11 +283,6 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { objectSummary += byteSizeToString(object.size) } - if cooked.RunningTally { - fileCount++ - sizeCount += object.size - } - glcm.Info(objectSummary) // No need to strip away from the name as the traverser has already done so. diff --git a/cmd/zt_list_test.go b/cmd/zt_list_test.go index 36fdd2e97..1f9feb7c8 100644 --- a/cmd/zt_list_test.go +++ b/cmd/zt_list_test.go @@ -49,6 +49,7 @@ func TestListVersions(t *testing.T) { rawContainerURLWithSAS := scenarioHelper{}.getSecondaryRawContainerURLWithSAS(a, containerName) raw := getDefaultListRawInput(rawContainerURLWithSAS.String()) raw.Properties = "VersionId" + raw.RunningTally = true runListAndVerify(a, raw, func(err error) { a.Nil(err) @@ -59,8 +60,16 @@ func TestListVersions(t *testing.T) { // check if info logs contain the correct version id for each blob msg := mockedLcm.GatherAllLogs(mockedLcm.infoLog) for i, m := range msg { - a.True(strings.Contains(m, blobsToInclude[i])) - a.True(strings.Contains(m, "VersionId: "+versions[blobsToInclude[i]])) + if i < 3 { // 0-2 will be blob names + version id + a.True(strings.Contains(m, blobsToInclude[i])) + a.True(strings.Contains(m, "VersionId: "+versions[blobsToInclude[i]])) + } + if i == 4 { // 4 will be file count + a.True(strings.Contains(m, "File count: 3")) + } + if i == 5 { // 5 will be file size + a.True(strings.Contains(m, "Total file size: 69.00 B")) + } } }) @@ -112,6 +121,7 @@ func TestListVersionsMultiVersions(t *testing.T) { rawContainerURLWithSAS := scenarioHelper{}.getSecondaryRawContainerURLWithSAS(a, containerName) raw := getDefaultListRawInput(rawContainerURLWithSAS.String()) raw.Properties = "VersionId" + raw.RunningTally = true runListAndVerify(a, raw, func(err error) { a.Nil(err) @@ -122,8 +132,16 @@ func TestListVersionsMultiVersions(t *testing.T) { // check if info logs contain the correct version id for each blob msg := mockedLcm.GatherAllLogs(mockedLcm.infoLog) for i, m := range msg { - a.True(strings.Contains(m, blobs[i])) - a.True(strings.Contains(m, versions[i])) + if i < 4 { // 0-3 will be blob names + version id + a.True(strings.Contains(m, blobs[i])) + a.True(strings.Contains(m, versions[i])) + } + if i == 5 { // 5 will be file count + a.True(strings.Contains(m, "File count: 3")) + } + if i == 6 { // 6 will be file size + a.True(strings.Contains(m, "Total file size: 69.00 B")) + } } }) From ada92d7c43d3e41df847d31a5e0ab6c04eb4bc2b Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Mon, 29 Jan 2024 16:12:09 -0500 Subject: [PATCH 05/12] handling edge case, handling edge case in test, updates to code --- cmd/list.go | 2 +- cmd/zt_list_test.go | 46 +++++++++++++++++++---------- cmd/zt_scenario_helpers_for_test.go | 12 ++++++++ 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/cmd/list.go b/cmd/list.go index 414fd0054..992b8d6cf 100755 --- a/cmd/list.go +++ b/cmd/list.go @@ -259,7 +259,7 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { processor := func(object StoredObject) error { if cooked.RunningTally { - if !(strings.Contains(objectSummary, object.name) && containsProperty(cooked.properties, versionId)) { + if !(strings.HasPrefix(objectSummary, object.relativePath) && containsProperty(cooked.properties, versionId)) { fileCount++ sizeCount += object.size } diff --git a/cmd/zt_list_test.go b/cmd/zt_list_test.go index 1f9feb7c8..6346aa2d4 100644 --- a/cmd/zt_list_test.go +++ b/cmd/zt_list_test.go @@ -1,7 +1,6 @@ package cmd import ( - "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/Azure/azure-storage-azcopy/v10/common" "github.com/stretchr/testify/assert" @@ -82,24 +81,24 @@ func TestListVersionsMultiVersions(t *testing.T) { containerClient, containerName := createNewContainer(a, bsc) defer deleteContainer(a, containerClient) - blobsToInclude := []string{"foo.txt", "sub1/dir2/bar.txt", "sub1/test/baz.txt"} + // testing how running tally will handle foo.txt vs foo/foo.txt, test/foo.txt + blobsToInclude := []string{"foo.txt", "foo/foo.txt", "test/foo.txt", "sub1/test/baz.txt"} scenarioHelper{}.generateBlobsFromList(a, containerClient, blobsToInclude, blockBlobDefaultData) a.NotNil(containerClient) - // make first blob have another version - bbClient := containerClient.NewBlockBlobClient(blobsToInclude[0]) - uploadResp, err := bbClient.Upload(ctx, streaming.NopCloser(strings.NewReader("Random random")), nil) - a.NoError(err) - a.NotNil(uploadResp.VersionID) + // make first two blobs have 1 additional version + blobsToVersion := []string{blobsToInclude[0], blobsToInclude[1]} + scenarioHelper{}.generateVersionsForBlobsFromList(a, containerClient, blobsToVersion) + a.NotNil(containerClient) - // confirm that container has 3 blobs + // confirm that container has 6 blobs (4 blobs, 2 versions) pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ Include: container.ListBlobsInclude{Versions: true}, }) list, err := pager.NextPage(ctx) a.NoError(err) a.NotNil(list.Segment.BlobItems) - a.Equal(4, len(list.Segment.BlobItems)) + a.Equal(6, len(list.Segment.BlobItems)) var blobs []string var versions []string @@ -132,17 +131,32 @@ func TestListVersionsMultiVersions(t *testing.T) { // check if info logs contain the correct version id for each blob msg := mockedLcm.GatherAllLogs(mockedLcm.infoLog) for i, m := range msg { - if i < 4 { // 0-3 will be blob names + version id - a.True(strings.Contains(m, blobs[i])) - a.True(strings.Contains(m, versions[i])) + if i < 6 { // 0-5 will be blob names + version id + a.True(contains(blobs, m, true)) + a.True(contains(versions, m, false)) } - if i == 5 { // 5 will be file count - a.True(strings.Contains(m, "File count: 3")) + if i == 7 { // 7 will be file count + a.True(strings.Contains(m, "File count: 4")) } - if i == 6 { // 6 will be file size - a.True(strings.Contains(m, "Total file size: 69.00 B")) + if i == 8 { // 8 will be file size + a.True(strings.Contains(m, "Total file size: 92.00 B")) } } }) } + +func contains(arr []string, msg string, isBlob bool) bool { + for _, a := range arr { + if isBlob { + if strings.HasPrefix(msg, a) { + return true + } + } else { + if strings.Contains(msg, a) { + return true + } + } + } + return false +} diff --git a/cmd/zt_scenario_helpers_for_test.go b/cmd/zt_scenario_helpers_for_test.go index 5e3cbad08..609412c6b 100644 --- a/cmd/zt_scenario_helpers_for_test.go +++ b/cmd/zt_scenario_helpers_for_test.go @@ -345,6 +345,18 @@ func (scenarioHelper) generateBlobsFromList(a *assert.Assertions, containerClien time.Sleep(time.Millisecond * 1050) } +func (scenarioHelper) generateVersionsForBlobsFromList(a *assert.Assertions, containerClient *container.Client, blobList []string) { + for _, blobName := range blobList { + blobClient := containerClient.NewBlockBlobClient(blobName) + uploadResp, err := blobClient.Upload(ctx, streaming.NopCloser(strings.NewReader("Random data "+generateBlobName())), nil) + a.NoError(err) + a.NotNil(uploadResp.VersionID) + } + + // sleep a bit so that the blobs' lmts are guaranteed to be in the past + time.Sleep(time.Millisecond * 1050) +} + func (scenarioHelper) generatePageBlobsFromList(a *assert.Assertions, containerClient *container.Client, blobList []string, data string) { for _, blobName := range blobList { // Create the blob (PUT blob) From 8554afbd4136fc343e33b06b9997ff86ecb53dc0 Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Tue, 30 Jan 2024 13:01:59 -0500 Subject: [PATCH 06/12] adding additional version to multiversions test --- cmd/zt_list_test.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/cmd/zt_list_test.go b/cmd/zt_list_test.go index 6346aa2d4..1e67df53c 100644 --- a/cmd/zt_list_test.go +++ b/cmd/zt_list_test.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/Azure/azure-storage-azcopy/v10/common" "github.com/stretchr/testify/assert" @@ -8,7 +9,7 @@ import ( "testing" ) -func TestListVersions(t *testing.T) { +func TestListVersionIdWithNoAdditionalVersions(t *testing.T) { a := assert.New(t) bsc := getSecondaryBlobServiceClient() // set up the container with single blob with 2 versions @@ -91,14 +92,24 @@ func TestListVersionsMultiVersions(t *testing.T) { scenarioHelper{}.generateVersionsForBlobsFromList(a, containerClient, blobsToVersion) a.NotNil(containerClient) - // confirm that container has 6 blobs (4 blobs, 2 versions) + // make first blob have 2 versions in total + blobClient := containerClient.NewBlockBlobClient(blobsToInclude[0]) + uploadResp, err := blobClient.Upload(ctx, streaming.NopCloser(strings.NewReader("Random")), nil) + a.NoError(err) + a.NotNil(uploadResp.VersionID) + a.NotNil(containerClient) + + // confirm that container has 7 blobs (4 blobs, 3 versions) + // foo.txt has two versions + // foo/foo.txt has one version + // test/foo.txt and sub1/test/baz.txt don't have any versions pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ Include: container.ListBlobsInclude{Versions: true}, }) list, err := pager.NextPage(ctx) a.NoError(err) a.NotNil(list.Segment.BlobItems) - a.Equal(6, len(list.Segment.BlobItems)) + a.Equal(7, len(list.Segment.BlobItems)) var blobs []string var versions []string @@ -131,14 +142,14 @@ func TestListVersionsMultiVersions(t *testing.T) { // check if info logs contain the correct version id for each blob msg := mockedLcm.GatherAllLogs(mockedLcm.infoLog) for i, m := range msg { - if i < 6 { // 0-5 will be blob names + version id + if i < 7 { // 0-5 will be blob names + version id a.True(contains(blobs, m, true)) a.True(contains(versions, m, false)) } - if i == 7 { // 7 will be file count + if i == 8 { // 8 will be file count a.True(strings.Contains(m, "File count: 4")) } - if i == 8 { // 8 will be file size + if i == 9 { // 9 will be file size a.True(strings.Contains(m, "Total file size: 92.00 B")) } } From 1702412d12589e8f012196088bcd7b7c7b8d968e Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Tue, 30 Jan 2024 14:29:36 -0500 Subject: [PATCH 07/12] updating running tally code to account for latest versions only --- cmd/list.go | 64 ++++++++++++++++++++++++++++++------ cmd/zt_list_test.go | 79 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 131 insertions(+), 12 deletions(-) diff --git a/cmd/list.go b/cmd/list.go index 992b8d6cf..bb1c6e4db 100755 --- a/cmd/list.go +++ b/cmd/list.go @@ -27,6 +27,7 @@ import ( "fmt" "strconv" "strings" + "time" "github.com/spf13/cobra" @@ -59,8 +60,15 @@ const ( leaseDuration validProperty = "LeaseDuration" leaseStatus validProperty = "LeaseStatus" archiveStatus validProperty = "ArchiveStatus" + + versionIdTimeFormat = "2006-01-02T15:04:05.9999999Z" ) +type versionIdObject struct { + versionId string + fileSize int64 +} + // containsProperty checks if the property array contains a valid property func containsProperty(properties []validProperty, prop validProperty) bool { for _, item := range properties { @@ -255,23 +263,16 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { var fileCount int64 = 0 var sizeCount int64 = 0 - var objectSummary string + objectVer := make(map[string]versionIdObject) processor := func(object StoredObject) error { - if cooked.RunningTally { - if !(strings.HasPrefix(objectSummary, object.relativePath) && containsProperty(cooked.properties, versionId)) { - fileCount++ - sizeCount += object.size - } - } - path := object.relativePath if object.entityType == common.EEntityType.Folder() { path += "/" // TODO: reviewer: same questions as for jobs status: OK to hard code direction of slash? OK to use trailing slash to distinguish dirs from files? } properties := "; " + cooked.processProperties(object) - objectSummary = path + properties + " Content Length: " + objectSummary := path + properties + " Content Length: " if level == level.Service() { objectSummary = object.ContainerName + "/" + objectSummary @@ -283,6 +284,42 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { objectSummary += byteSizeToString(object.size) } + if cooked.RunningTally { + if containsProperty(cooked.properties, versionId) { + // get new version id object + updatedVersionId := versionIdObject{ + versionId: object.blobVersionID, + fileSize: object.size, + } + + // there exists a current version id of the object + if currentVersionId, ok := objectVer[object.relativePath]; ok { + // get current version id time + currentVid, err := time.Parse(versionIdTimeFormat, currentVersionId.versionId) + if err != nil { + fmt.Errorf("unable to parse version id into time format: %s", err.Error()) + } + + // get new version id time + newVid, err := time.Parse(versionIdTimeFormat, object.blobVersionID) + if err != nil { + fmt.Errorf("unable to parse version id into time format: %s", err.Error()) + } + + // if new vid came after the current vid, then it is the latest version + // update the objectVer with the latest version + if newVid.After(currentVid) { + objectVer[object.relativePath] = updatedVersionId + } + } else { + objectVer[object.relativePath] = updatedVersionId + } + } else { + fileCount++ + sizeCount += object.size + } + } + glcm.Info(objectSummary) // No need to strip away from the name as the traverser has already done so. @@ -296,6 +333,15 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { } if cooked.RunningTally { + if containsProperty(cooked.properties, versionId) { + // get file count by getting length of objectVer + fileCount = int64(len(objectVer)) + + // get size count by incrementing through objectVer and adding all values of versionIdObject.fileSize + for _, val := range objectVer { + sizeCount += val.fileSize + } + } glcm.Info("") glcm.Info("File count: " + strconv.Itoa(int(fileCount))) diff --git a/cmd/zt_list_test.go b/cmd/zt_list_test.go index 1e67df53c..fa6f0c925 100644 --- a/cmd/zt_list_test.go +++ b/cmd/zt_list_test.go @@ -142,15 +142,88 @@ func TestListVersionsMultiVersions(t *testing.T) { // check if info logs contain the correct version id for each blob msg := mockedLcm.GatherAllLogs(mockedLcm.infoLog) for i, m := range msg { - if i < 7 { // 0-5 will be blob names + version id + if i < 7 { // 0-6 will be blob names + version id a.True(contains(blobs, m, true)) a.True(contains(versions, m, false)) } if i == 8 { // 8 will be file count a.True(strings.Contains(m, "File count: 4")) } - if i == 9 { // 9 will be file size - a.True(strings.Contains(m, "Total file size: 92.00 B")) + if i == 9 { // 9 will be file size of latest versions (should be 106) + a.True(strings.Contains(m, "Total file size: 106.00 B")) + } + } + }) + +} + +func TestListVersionsMultiVersionsNoPropFlag(t *testing.T) { + a := assert.New(t) + bsc := getSecondaryBlobServiceClient() + // set up the container with single blob with 2 versions + containerClient, containerName := createNewContainer(a, bsc) + defer deleteContainer(a, containerClient) + + // testing how running tally will handle foo.txt vs foo/foo.txt, test/foo.txt + blobsToInclude := []string{"foo.txt", "foo/foo.txt", "test/foo.txt", "sub1/test/baz.txt"} + scenarioHelper{}.generateBlobsFromList(a, containerClient, blobsToInclude, blockBlobDefaultData) + a.NotNil(containerClient) + + // make first two blobs have 1 additional version + blobsToVersion := []string{blobsToInclude[0], blobsToInclude[1]} + scenarioHelper{}.generateVersionsForBlobsFromList(a, containerClient, blobsToVersion) + a.NotNil(containerClient) + + // make first blob have 2 versions in total + blobClient := containerClient.NewBlockBlobClient(blobsToInclude[0]) + uploadResp, err := blobClient.Upload(ctx, streaming.NopCloser(strings.NewReader("Random")), nil) + a.NoError(err) + a.NotNil(uploadResp.VersionID) + a.NotNil(containerClient) + + // confirm that container has 7 blobs (4 blobs, 3 versions) + // foo.txt has two versions + // foo/foo.txt has one version + // test/foo.txt and sub1/test/baz.txt don't have any versions + pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ + Include: container.ListBlobsInclude{Versions: true}, + }) + list, err := pager.NextPage(ctx) + a.NoError(err) + a.NotNil(list.Segment.BlobItems) + a.Equal(7, len(list.Segment.BlobItems)) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + mockedLcm := mockedLifecycleManager{infoLog: make(chan string, 50)} + mockedLcm.SetOutputFormat(common.EOutputFormat.Text()) // text format + glcm = &mockedLcm + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getSecondaryRawContainerURLWithSAS(a, containerName) + raw := getDefaultListRawInput(rawContainerURLWithSAS.String()) + raw.RunningTally = true + + runListAndVerify(a, raw, func(err error) { + a.Nil(err) + + // validate that the no transfers were scheduled + a.Nil(mockedRPC.transfers) + + // check if info logs contain the correct version id for each blob + msg := mockedLcm.GatherAllLogs(mockedLcm.infoLog) + for i, m := range msg { + if i < 4 { // 0-3 is blob + a.True(contains(blobsToInclude, m, true)) + } + if i == 5 { // 5 will be file count + a.True(strings.Contains(m, "File count: 4")) + } + if i == 6 { // 6 will be file size (should be 116) + a.True(strings.Contains(m, "Total file size: 116.00 B")) } } }) From 8498cf31184dbc20e9cc69551c1d6b81505aa695 Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:01:35 -0500 Subject: [PATCH 08/12] cleaning up code --- cmd/list.go | 10 ++-------- cmd/zt_list_test.go | 8 ++++---- cmd/zt_scenario_helpers_for_test.go | 2 +- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/cmd/list.go b/cmd/list.go index bb1c6e4db..e0437c913 100755 --- a/cmd/list.go +++ b/cmd/list.go @@ -295,16 +295,10 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { // there exists a current version id of the object if currentVersionId, ok := objectVer[object.relativePath]; ok { // get current version id time - currentVid, err := time.Parse(versionIdTimeFormat, currentVersionId.versionId) - if err != nil { - fmt.Errorf("unable to parse version id into time format: %s", err.Error()) - } + currentVid, _ := time.Parse(versionIdTimeFormat, currentVersionId.versionId) // get new version id time - newVid, err := time.Parse(versionIdTimeFormat, object.blobVersionID) - if err != nil { - fmt.Errorf("unable to parse version id into time format: %s", err.Error()) - } + newVid, _ := time.Parse(versionIdTimeFormat, object.blobVersionID) // if new vid came after the current vid, then it is the latest version // update the objectVer with the latest version diff --git a/cmd/zt_list_test.go b/cmd/zt_list_test.go index fa6f0c925..d6ecfd502 100644 --- a/cmd/zt_list_test.go +++ b/cmd/zt_list_test.go @@ -149,8 +149,8 @@ func TestListVersionsMultiVersions(t *testing.T) { if i == 8 { // 8 will be file count a.True(strings.Contains(m, "File count: 4")) } - if i == 9 { // 9 will be file size of latest versions (should be 106) - a.True(strings.Contains(m, "Total file size: 106.00 B")) + if i == 9 { // 9 will be file size of latest versions (should be 64) + a.True(strings.Contains(m, "Total file size: 64.00 B")) } } }) @@ -222,8 +222,8 @@ func TestListVersionsMultiVersionsNoPropFlag(t *testing.T) { if i == 5 { // 5 will be file count a.True(strings.Contains(m, "File count: 4")) } - if i == 6 { // 6 will be file size (should be 116) - a.True(strings.Contains(m, "Total file size: 116.00 B")) + if i == 6 { // 6 will be file size (should be 64) + a.True(strings.Contains(m, "Total file size: 64.00 B")) } } }) diff --git a/cmd/zt_scenario_helpers_for_test.go b/cmd/zt_scenario_helpers_for_test.go index 609412c6b..03eeeb5c0 100644 --- a/cmd/zt_scenario_helpers_for_test.go +++ b/cmd/zt_scenario_helpers_for_test.go @@ -348,7 +348,7 @@ func (scenarioHelper) generateBlobsFromList(a *assert.Assertions, containerClien func (scenarioHelper) generateVersionsForBlobsFromList(a *assert.Assertions, containerClient *container.Client, blobList []string) { for _, blobName := range blobList { blobClient := containerClient.NewBlockBlobClient(blobName) - uploadResp, err := blobClient.Upload(ctx, streaming.NopCloser(strings.NewReader("Random data "+generateBlobName())), nil) + uploadResp, err := blobClient.Upload(ctx, streaming.NopCloser(strings.NewReader("Random data ")), nil) a.NoError(err) a.NotNil(uploadResp.VersionID) } From ede3c28913e8c193ab77d4f8052ac6c410694cb0 Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Mon, 12 Feb 2024 17:17:14 -0500 Subject: [PATCH 09/12] call containsProperty once --- cmd/list.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/list.go b/cmd/list.go index e0437c913..e28d08d1a 100755 --- a/cmd/list.go +++ b/cmd/list.go @@ -255,7 +255,10 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { } } - traverser, err := InitResourceTraverser(source, cooked.location, &ctx, &credentialInfo, common.ESymlinkHandlingType.Skip(), nil, true, true, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, nil, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), common.LogNone, common.CpkOptions{}, nil, false, cooked.trailingDot, nil, nil, containsProperty(cooked.properties, versionId)) + // check if user wants to get version id + shouldGetVersionId := containsProperty(cooked.properties, versionId) + + traverser, err := InitResourceTraverser(source, cooked.location, &ctx, &credentialInfo, common.ESymlinkHandlingType.Skip(), nil, true, true, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, nil, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), common.LogNone, common.CpkOptions{}, nil, false, cooked.trailingDot, nil, nil, shouldGetVersionId) if err != nil { return fmt.Errorf("failed to initialize traverser: %s", err.Error()) @@ -285,7 +288,7 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { } if cooked.RunningTally { - if containsProperty(cooked.properties, versionId) { + if shouldGetVersionId { // get new version id object updatedVersionId := versionIdObject{ versionId: object.blobVersionID, @@ -327,7 +330,7 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { } if cooked.RunningTally { - if containsProperty(cooked.properties, versionId) { + if shouldGetVersionId { // get file count by getting length of objectVer fileCount = int64(len(objectVer)) From 0ac41f803ef3c0a9c6106c01cde33ab0fa22fe41 Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Mon, 12 Feb 2024 17:44:39 -0500 Subject: [PATCH 10/12] making sizeCount and fileCount update dynamically and smartly --- cmd/list.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/cmd/list.go b/cmd/list.go index e28d08d1a..e6a1d6871 100755 --- a/cmd/list.go +++ b/cmd/list.go @@ -305,16 +305,19 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { // if new vid came after the current vid, then it is the latest version // update the objectVer with the latest version + // we will also remove sizeCount and fileCount of current object, allowing + // the updated sizeCount and fileCount to be added at line 320 if newVid.After(currentVid) { + sizeCount -= currentVersionId.fileSize // remove size of current object + fileCount-- // remove current object file count objectVer[object.relativePath] = updatedVersionId } } else { objectVer[object.relativePath] = updatedVersionId } - } else { - fileCount++ - sizeCount += object.size } + fileCount++ + sizeCount += object.size } glcm.Info(objectSummary) @@ -330,15 +333,6 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { } if cooked.RunningTally { - if shouldGetVersionId { - // get file count by getting length of objectVer - fileCount = int64(len(objectVer)) - - // get size count by incrementing through objectVer and adding all values of versionIdObject.fileSize - for _, val := range objectVer { - sizeCount += val.fileSize - } - } glcm.Info("") glcm.Info("File count: " + strconv.Itoa(int(fileCount))) From 65980b39be1dfdc540d1e3592d6f7be5723f6aae Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Wed, 14 Feb 2024 23:11:28 -0500 Subject: [PATCH 11/12] addressing comments --- cmd/list.go | 10 +-- cmd/zt_list_test.go | 103 ++++++++++++++++++++++++---- cmd/zt_scenario_helpers_for_test.go | 6 +- 3 files changed, 97 insertions(+), 22 deletions(-) diff --git a/cmd/list.go b/cmd/list.go index e6a1d6871..4a6437578 100755 --- a/cmd/list.go +++ b/cmd/list.go @@ -64,11 +64,6 @@ const ( versionIdTimeFormat = "2006-01-02T15:04:05.9999999Z" ) -type versionIdObject struct { - versionId string - fileSize int64 -} - // containsProperty checks if the property array contains a valid property func containsProperty(properties []validProperty, prop validProperty) bool { for _, item := range properties { @@ -266,6 +261,11 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { var fileCount int64 = 0 var sizeCount int64 = 0 + + type versionIdObject struct { + versionId string + fileSize int64 + } objectVer := make(map[string]versionIdObject) processor := func(object StoredObject) error { diff --git a/cmd/zt_list_test.go b/cmd/zt_list_test.go index d6ecfd502..81f9ed73e 100644 --- a/cmd/zt_list_test.go +++ b/cmd/zt_list_test.go @@ -12,7 +12,7 @@ import ( func TestListVersionIdWithNoAdditionalVersions(t *testing.T) { a := assert.New(t) bsc := getSecondaryBlobServiceClient() - // set up the container with single blob with 2 versions + containerClient, containerName := createNewContainer(a, bsc) defer deleteContainer(a, containerClient) @@ -43,7 +43,6 @@ func TestListVersionIdWithNoAdditionalVersions(t *testing.T) { mockedLcm := mockedLifecycleManager{infoLog: make(chan string, 50)} mockedLcm.SetOutputFormat(common.EOutputFormat.Text()) // text format - glcm = &mockedLcm // construct the raw input to simulate user input rawContainerURLWithSAS := scenarioHelper{}.getSecondaryRawContainerURLWithSAS(a, containerName) @@ -73,12 +72,38 @@ func TestListVersionIdWithNoAdditionalVersions(t *testing.T) { } }) + // test json output + mockedLcm = mockedLifecycleManager{infoLog: make(chan string, 50)} + mockedLcm.SetOutputFormat(common.EOutputFormat.Json()) // json format + + runListAndVerify(a, raw, func(err error) { + a.Nil(err) + + // validate that the no transfers were scheduled + a.Nil(mockedRPC.transfers) + + // check if info logs contain the correct version id for each blob + msg := mockedLcm.GatherAllLogs(mockedLcm.infoLog) + for i, m := range msg { + if i < 3 { // 0-2 will be blob names + version id + a.True(strings.Contains(m, blobsToInclude[i])) + a.True(strings.Contains(m, "VersionId: "+versions[blobsToInclude[i]])) + } + if i == 4 { // 4 will be file count + a.True(strings.Contains(m, "File count: 3")) + } + if i == 5 { // 5 will be file size + a.True(strings.Contains(m, "Total file size: 69.00 B")) + } + } + }) + } func TestListVersionsMultiVersions(t *testing.T) { a := assert.New(t) bsc := getSecondaryBlobServiceClient() - // set up the container with single blob with 2 versions + containerClient, containerName := createNewContainer(a, bsc) defer deleteContainer(a, containerClient) @@ -89,12 +114,13 @@ func TestListVersionsMultiVersions(t *testing.T) { // make first two blobs have 1 additional version blobsToVersion := []string{blobsToInclude[0], blobsToInclude[1]} - scenarioHelper{}.generateVersionsForBlobsFromList(a, containerClient, blobsToVersion) + randomStrings := []string{"random-1", "random-two"} + scenarioHelper{}.generateVersionsForBlobsFromList(a, containerClient, blobsToVersion, randomStrings) a.NotNil(containerClient) // make first blob have 2 versions in total blobClient := containerClient.NewBlockBlobClient(blobsToInclude[0]) - uploadResp, err := blobClient.Upload(ctx, streaming.NopCloser(strings.NewReader("Random")), nil) + uploadResp, err := blobClient.Upload(ctx, streaming.NopCloser(strings.NewReader("random-three-3")), nil) a.NoError(err) a.NotNil(uploadResp.VersionID) a.NotNil(containerClient) @@ -125,7 +151,6 @@ func TestListVersionsMultiVersions(t *testing.T) { mockedLcm := mockedLifecycleManager{infoLog: make(chan string, 50)} mockedLcm.SetOutputFormat(common.EOutputFormat.Text()) // text format - glcm = &mockedLcm // construct the raw input to simulate user input rawContainerURLWithSAS := scenarioHelper{}.getSecondaryRawContainerURLWithSAS(a, containerName) @@ -149,8 +174,34 @@ func TestListVersionsMultiVersions(t *testing.T) { if i == 8 { // 8 will be file count a.True(strings.Contains(m, "File count: 4")) } - if i == 9 { // 9 will be file size of latest versions (should be 64) - a.True(strings.Contains(m, "Total file size: 64.00 B")) + if i == 9 { // 9 will be file size of latest versions (should be 70.00 B) + a.True(strings.Contains(m, "Total file size: 70.00 B")) + } + } + }) + + // test json output + mockedLcm = mockedLifecycleManager{infoLog: make(chan string, 50)} + mockedLcm.SetOutputFormat(common.EOutputFormat.Json()) // json format + + runListAndVerify(a, raw, func(err error) { + a.Nil(err) + + // validate that the no transfers were scheduled + a.Nil(mockedRPC.transfers) + + // check if info logs contain the correct version id for each blob + msg := mockedLcm.GatherAllLogs(mockedLcm.infoLog) + for i, m := range msg { + if i < 7 { // 0-6 will be blob names + version id + a.True(contains(blobs, m, true)) + a.True(contains(versions, m, false)) + } + if i == 8 { // 8 will be file count + a.True(strings.Contains(m, "File count: 4")) + } + if i == 9 { // 9 will be file size of latest versions (should be 70.00 B) + a.True(strings.Contains(m, "Total file size: 70.00 B")) } } }) @@ -160,7 +211,7 @@ func TestListVersionsMultiVersions(t *testing.T) { func TestListVersionsMultiVersionsNoPropFlag(t *testing.T) { a := assert.New(t) bsc := getSecondaryBlobServiceClient() - // set up the container with single blob with 2 versions + containerClient, containerName := createNewContainer(a, bsc) defer deleteContainer(a, containerClient) @@ -171,12 +222,13 @@ func TestListVersionsMultiVersionsNoPropFlag(t *testing.T) { // make first two blobs have 1 additional version blobsToVersion := []string{blobsToInclude[0], blobsToInclude[1]} - scenarioHelper{}.generateVersionsForBlobsFromList(a, containerClient, blobsToVersion) + randomStrings := []string{"random-1", "random-two"} + scenarioHelper{}.generateVersionsForBlobsFromList(a, containerClient, blobsToVersion, randomStrings) a.NotNil(containerClient) // make first blob have 2 versions in total blobClient := containerClient.NewBlockBlobClient(blobsToInclude[0]) - uploadResp, err := blobClient.Upload(ctx, streaming.NopCloser(strings.NewReader("Random")), nil) + uploadResp, err := blobClient.Upload(ctx, streaming.NopCloser(strings.NewReader("random-three-3")), nil) a.NoError(err) a.NotNil(uploadResp.VersionID) a.NotNil(containerClient) @@ -200,7 +252,6 @@ func TestListVersionsMultiVersionsNoPropFlag(t *testing.T) { mockedLcm := mockedLifecycleManager{infoLog: make(chan string, 50)} mockedLcm.SetOutputFormat(common.EOutputFormat.Text()) // text format - glcm = &mockedLcm // construct the raw input to simulate user input rawContainerURLWithSAS := scenarioHelper{}.getSecondaryRawContainerURLWithSAS(a, containerName) @@ -222,12 +273,36 @@ func TestListVersionsMultiVersionsNoPropFlag(t *testing.T) { if i == 5 { // 5 will be file count a.True(strings.Contains(m, "File count: 4")) } - if i == 6 { // 6 will be file size (should be 64) - a.True(strings.Contains(m, "Total file size: 64.00 B")) + if i == 6 { // 6 will be file size (should be 70 B) + a.True(strings.Contains(m, "Total file size: 70.00 B")) } } }) + // test json output + mockedLcm = mockedLifecycleManager{infoLog: make(chan string, 50)} + mockedLcm.SetOutputFormat(common.EOutputFormat.Json()) // json format + + runListAndVerify(a, raw, func(err error) { + a.Nil(err) + + // validate that the no transfers were scheduled + a.Nil(mockedRPC.transfers) + + // check if info logs contain the correct version id for each blob + msg := mockedLcm.GatherAllLogs(mockedLcm.infoLog) + for i, m := range msg { + if i < 4 { // 0-3 is blob + a.True(contains(blobsToInclude, m, true)) + } + if i == 5 { // 5 will be file count + a.True(strings.Contains(m, "File count: 4")) + } + if i == 6 { // 6 will be file size (should be 70 B) + a.True(strings.Contains(m, "Total file size: 70.00 B")) + } + } + }) } func contains(arr []string, msg string, isBlob bool) bool { diff --git a/cmd/zt_scenario_helpers_for_test.go b/cmd/zt_scenario_helpers_for_test.go index 03eeeb5c0..0e00f0202 100644 --- a/cmd/zt_scenario_helpers_for_test.go +++ b/cmd/zt_scenario_helpers_for_test.go @@ -345,10 +345,10 @@ func (scenarioHelper) generateBlobsFromList(a *assert.Assertions, containerClien time.Sleep(time.Millisecond * 1050) } -func (scenarioHelper) generateVersionsForBlobsFromList(a *assert.Assertions, containerClient *container.Client, blobList []string) { - for _, blobName := range blobList { +func (scenarioHelper) generateVersionsForBlobsFromList(a *assert.Assertions, containerClient *container.Client, blobList []string, randomData []string) { + for i, blobName := range blobList { blobClient := containerClient.NewBlockBlobClient(blobName) - uploadResp, err := blobClient.Upload(ctx, streaming.NopCloser(strings.NewReader("Random data ")), nil) + uploadResp, err := blobClient.Upload(ctx, streaming.NopCloser(strings.NewReader(randomData[i])), nil) a.NoError(err) a.NotNil(uploadResp.VersionID) } From 4564d1c7aa05727f298747236a745f3f4f43d7eb Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Wed, 14 Feb 2024 23:17:42 -0500 Subject: [PATCH 12/12] minor update to test --- cmd/zt_list_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/zt_list_test.go b/cmd/zt_list_test.go index 81f9ed73e..b2df43118 100644 --- a/cmd/zt_list_test.go +++ b/cmd/zt_list_test.go @@ -43,6 +43,7 @@ func TestListVersionIdWithNoAdditionalVersions(t *testing.T) { mockedLcm := mockedLifecycleManager{infoLog: make(chan string, 50)} mockedLcm.SetOutputFormat(common.EOutputFormat.Text()) // text format + glcm = &mockedLcm // construct the raw input to simulate user input rawContainerURLWithSAS := scenarioHelper{}.getSecondaryRawContainerURLWithSAS(a, containerName) @@ -75,6 +76,7 @@ func TestListVersionIdWithNoAdditionalVersions(t *testing.T) { // test json output mockedLcm = mockedLifecycleManager{infoLog: make(chan string, 50)} mockedLcm.SetOutputFormat(common.EOutputFormat.Json()) // json format + glcm = &mockedLcm runListAndVerify(a, raw, func(err error) { a.Nil(err) @@ -151,6 +153,7 @@ func TestListVersionsMultiVersions(t *testing.T) { mockedLcm := mockedLifecycleManager{infoLog: make(chan string, 50)} mockedLcm.SetOutputFormat(common.EOutputFormat.Text()) // text format + glcm = &mockedLcm // construct the raw input to simulate user input rawContainerURLWithSAS := scenarioHelper{}.getSecondaryRawContainerURLWithSAS(a, containerName) @@ -183,6 +186,7 @@ func TestListVersionsMultiVersions(t *testing.T) { // test json output mockedLcm = mockedLifecycleManager{infoLog: make(chan string, 50)} mockedLcm.SetOutputFormat(common.EOutputFormat.Json()) // json format + glcm = &mockedLcm runListAndVerify(a, raw, func(err error) { a.Nil(err) @@ -252,6 +256,7 @@ func TestListVersionsMultiVersionsNoPropFlag(t *testing.T) { mockedLcm := mockedLifecycleManager{infoLog: make(chan string, 50)} mockedLcm.SetOutputFormat(common.EOutputFormat.Text()) // text format + glcm = &mockedLcm // construct the raw input to simulate user input rawContainerURLWithSAS := scenarioHelper{}.getSecondaryRawContainerURLWithSAS(a, containerName) @@ -282,6 +287,7 @@ func TestListVersionsMultiVersionsNoPropFlag(t *testing.T) { // test json output mockedLcm = mockedLifecycleManager{infoLog: make(chan string, 50)} mockedLcm.SetOutputFormat(common.EOutputFormat.Json()) // json format + glcm = &mockedLcm runListAndVerify(a, raw, func(err error) { a.Nil(err)