[GitHub] azcopy list with versionids does not show version id

cmd/copyEnumeratorInit.go

@@ -72,7 +72,7 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
 	jobPartOrder.S2SPreserveBlobTags = cca.S2sPreserveBlobTags
 
 	dest := cca.FromTo.To()
-	traverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &srcCredInfo, cca.SymlinkHandling, cca.ListOfFilesChannel, cca.Recursive, getRemoteProperties, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, cca.S2sPreserveBlobTags, common.ESyncHashType.None(), cca.preservePermissions, azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, &dest, cca.excludeContainer)
+	traverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &srcCredInfo, cca.SymlinkHandling, cca.ListOfFilesChannel, cca.Recursive, getRemoteProperties, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, cca.S2sPreserveBlobTags, common.ESyncHashType.None(), cca.preservePermissions, azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, &dest, cca.excludeContainer, false)
 
 	if err != nil {
 		return nil, err
@@ -343,7 +343,7 @@ func (cca *CookedCopyCmdArgs) isDestDirectory(dst common.ResourceString, ctx *co
 		return false
 	}
 
-	rt, err := InitResourceTraverser(dst, cca.FromTo.To(), ctx, &dstCredInfo, common.ESymlinkHandlingType.Skip(), nil, false, false, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), cca.preservePermissions, common.LogNone, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil, cca.excludeContainer)
+	rt, err := InitResourceTraverser(dst, cca.FromTo.To(), ctx, &dstCredInfo, common.ESymlinkHandlingType.Skip(), nil, false, false, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), cca.preservePermissions, common.LogNone, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil, cca.excludeContainer, false)
 
 	if err != nil {
 		return false

cmd/copyEnumeratorInit_test.go

@@ -162,4 +162,4 @@ func TestValidateSourceWithWildCard(t *testing.T) {
 	err := cca.validateSourceDir(blobTraverser)
 	a.Nil(err)
 	a.False(cca.IsSourceDir)
-}
+}

cmd/list.go

@@ -61,6 +61,16 @@ const (
 	archiveStatus    validProperty = "ArchiveStatus"
 )
 
+// containsProperty checks if the property array contains a valid property
+func containsProperty(properties []validProperty, prop validProperty) bool {
+	for _, item := range properties {
+		if item == prop {
+			return true
+		}
+	}
+	return false
+}
+
 // validProperties returns an array of possible values for the validProperty const type.
 func validProperties() []validProperty {
 	return []validProperty{lastModifiedTime, versionId, blobType, blobAccessTier,
@@ -237,23 +247,31 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) {
 		}
 	}
 
-	traverser, err := InitResourceTraverser(source, cooked.location, &ctx, &credentialInfo, common.ESymlinkHandlingType.Skip(), nil, true, true, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, nil, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), common.LogNone, common.CpkOptions{}, nil, false, cooked.trailingDot, nil, nil)
+	traverser, err := InitResourceTraverser(source, cooked.location, &ctx, &credentialInfo, common.ESymlinkHandlingType.Skip(), nil, true, true, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, nil, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), common.LogNone, common.CpkOptions{}, nil, false, cooked.trailingDot, nil, nil, containsProperty(cooked.properties, versionId))
 
 	if err != nil {
 		return fmt.Errorf("failed to initialize traverser: %s", err.Error())
 	}
 
 	var fileCount int64 = 0
 	var sizeCount int64 = 0
+	var objectSummary string
 
 	processor := func(object StoredObject) error {
+		if cooked.RunningTally {
+			if !(strings.HasPrefix(objectSummary, object.relativePath) && containsProperty(cooked.properties, versionId)) {
+				fileCount++
+				sizeCount += object.size
+			}
+		}
+
 		path := object.relativePath
 		if object.entityType == common.EEntityType.Folder() {
 			path += "/" // TODO: reviewer: same questions as for jobs status: OK to hard code direction of slash? OK to use trailing slash to distinguish dirs from files?
 		}
 
 		properties := "; " + cooked.processProperties(object)
-		objectSummary := path + properties + " Content Length: "
+		objectSummary = path + properties + " Content Length: "
 
 		if level == level.Service() {
 			objectSummary = object.ContainerName + "/" + objectSummary
@@ -265,11 +283,6 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) {
 			objectSummary += byteSizeToString(object.size)
 		}
 
-		if cooked.RunningTally {
-			fileCount++
-			sizeCount += object.size
-		}
-
 		glcm.Info(objectSummary)
 
 		// No need to strip away from the name as the traverser has already done so.

cmd/removeEnumerator.go

@@ -50,7 +50,7 @@ func newRemoveEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator, er
 	ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion)
 
 	// Include-path is handled by ListOfFilesChannel.
-	sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo, common.ESymlinkHandlingType.Skip(), cca.ListOfFilesChannel, cca.Recursive, true, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil, cca.excludeContainer)
+	sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo, common.ESymlinkHandlingType.Skip(), cca.ListOfFilesChannel, cca.Recursive, true, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil, cca.excludeContainer, false)
 
 	// report failure to create traverser
 	if err != nil {

cmd/setPropertiesEnumerator.go

@@ -46,7 +46,7 @@ func setPropertiesEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator
 	}
 
 	// Include-path is handled by ListOfFilesChannel.
-	sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo, common.ESymlinkHandlingType.Preserve(), cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil, cca.excludeContainer)
+	sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo, common.ESymlinkHandlingType.Preserve(), cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil, cca.excludeContainer, false)
 
 	// report failure to create traverser
 	if err != nil {

cmd/syncEnumerator.go

@@ -64,7 +64,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s
 		if entityType == common.EEntityType.File() {
 			atomic.AddUint64(&cca.atomicSourceFilesScanned, 1)
 		}
-	}, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity, cca.cpkOptions, nil, false, cca.trailingDot, &dest, nil)
+	}, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity, cca.cpkOptions, nil, false, cca.trailingDot, &dest, nil, false)
 
 	if err != nil {
 		return nil, err
@@ -85,7 +85,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s
 		if entityType == common.EEntityType.File() {
 			atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1)
 		}
-	}, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity, cca.cpkOptions, nil, false, cca.trailingDot, nil, nil)
+	}, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity, cca.cpkOptions, nil, false, cca.trailingDot, nil, nil, false)
 	if err != nil {
 		return nil, err
 	}

cmd/zc_enumerator.go

@@ -334,7 +334,7 @@ type enumerationCounterFunc func(entityType common.EntityType)
 // errorOnDirWOutRecursive is used by copy.
 // If errorChannel is non-nil, all errors encountered during enumeration will be conveyed through this channel.
 // To avoid slowdowns, use a buffered channel of enough capacity.
-func InitResourceTraverser(resource common.ResourceString, location common.Location, ctx *context.Context, credential *common.CredentialInfo, symlinkHandling common.SymlinkHandlingType, listOfFilesChannel chan string, recursive, getProperties, includeDirectoryStubs bool, permanentDeleteOption common.PermanentDeleteOption, incrementEnumerationCounter enumerationCounterFunc, listOfVersionIds chan string, s2sPreserveBlobTags bool, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption, logLevel common.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo, stripTopDir bool, trailingDot common.TrailingDotOption, destination *common.Location, excludeContainerNames []string) (ResourceTraverser, error) {
+func InitResourceTraverser(resource common.ResourceString, location common.Location, ctx *context.Context, credential *common.CredentialInfo, symlinkHandling common.SymlinkHandlingType, listOfFilesChannel chan string, recursive, getProperties, includeDirectoryStubs bool, permanentDeleteOption common.PermanentDeleteOption, incrementEnumerationCounter enumerationCounterFunc, listOfVersionIds chan string, s2sPreserveBlobTags bool, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption, logLevel common.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo, stripTopDir bool, trailingDot common.TrailingDotOption, destination *common.Location, excludeContainerNames []string, includeVersionsList bool) (ResourceTraverser, error) {
 	var output ResourceTraverser
 
 	var includeDeleted bool
@@ -353,6 +353,10 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat
 		includeVersion = true
 	}
 
+	// print out version id when using azcopy list
+	if includeVersionsList {
+		includeVersion = true
+	}
 	// Clean up the resource if it's a local path
 	if location == common.ELocation.Local() {
 		resource = common.ResourceString{Value: cleanLocalPath(resource.ValueLocal())}

cmd/zc_traverser_blob.go

@@ -476,7 +476,7 @@ func (t *blobTraverser) createStoredObjectForBlob(preprocessor objectMorpher, bl
 	object.blobDeleted = common.IffNotNil(blobInfo.Deleted, false)
 	if t.includeDeleted && t.includeSnapshot {
 		object.blobSnapshotID = common.IffNotNil(blobInfo.Snapshot, "")
-	} else if t.includeDeleted && t.includeVersion && blobInfo.VersionID != nil {
+	} else if t.includeVersion && blobInfo.VersionID != nil {
 		object.blobVersionID = common.IffNotNil(blobInfo.VersionID, "")
 	}
 	return object

cmd/zc_traverser_list.go

@@ -107,7 +107,7 @@ func newListTraverser(parent common.ResourceString, parentType common.Location,
 		}
 
 		// Construct a traverser that goes through the child
-		traverser, err := InitResourceTraverser(source, parentType, ctx, credential, handleSymlinks, nil, recursive, getProperties, includeDirectoryStubs, common.EPermanentDeleteOption.None(), incrementEnumerationCounter, nil, s2sPreserveBlobTags, syncHashType, preservePermissions, logLevel, cpkOptions, nil, false, trailingDot, destination, nil)
+		traverser, err := InitResourceTraverser(source, parentType, ctx, credential, handleSymlinks, nil, recursive, getProperties, includeDirectoryStubs, common.EPermanentDeleteOption.None(), incrementEnumerationCounter, nil, s2sPreserveBlobTags, syncHashType, preservePermissions, logLevel, cpkOptions, nil, false, trailingDot, destination, nil, false)
 		if err != nil {
 			return nil, err
 		}

cmd/zt_generic_traverser_test.go

@@ -104,6 +104,7 @@ func TestLocalWildcardOverlap(t *testing.T) {
 		common.ETrailingDotOption.Enable(),
 		nil,
 		nil,
+		false,
 	)
 	a.Nil(err)
 

cmd/zt_list_test.go

@@ -0,0 +1,162 @@
+package cmd
+
+import (
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
+	"github.com/Azure/azure-storage-azcopy/v10/common"
+	"github.com/stretchr/testify/assert"
+	"strings"
+	"testing"
+)
+
+func TestListVersions(t *testing.T) {
+	a := assert.New(t)
+	bsc := getSecondaryBlobServiceClient()
+	// set up the container with single blob with 2 versions
+	containerClient, containerName := createNewContainer(a, bsc)
+	defer deleteContainer(a, containerClient)
+
+	blobsToInclude := []string{"AzURE2021.jpeg", "sub1/dir2/HELLO-4.txt", "sub1/test/testing.txt"}
+	scenarioHelper{}.generateBlobsFromList(a, containerClient, blobsToInclude, blockBlobDefaultData)
+	a.NotNil(containerClient)
+
+	// get dictionary/map of blob: version id
+	versions := make(map[string]string)
+	for _, blob := range blobsToInclude {
+		props, err := containerClient.NewBlockBlobClient(blob).GetProperties(ctx, nil)
+		a.NoError(err)
+
+		versions[blob] = *props.VersionID
+	}
+
+	// confirm that container has 3 blobs
+	pager := containerClient.NewListBlobsFlatPager(nil)
+	list, err := pager.NextPage(ctx)
+	a.NoError(err)
+	a.NotNil(list.Segment.BlobItems)
+	a.Equal(3, len(list.Segment.BlobItems))
+
+	// set up interceptor
+	mockedRPC := interceptor{}
+	Rpc = mockedRPC.intercept
+	mockedRPC.init()
+
+	mockedLcm := mockedLifecycleManager{infoLog: make(chan string, 50)}
+	mockedLcm.SetOutputFormat(common.EOutputFormat.Text()) // text format
+	glcm = &mockedLcm
+
+	// construct the raw input to simulate user input
+	rawContainerURLWithSAS := scenarioHelper{}.getSecondaryRawContainerURLWithSAS(a, containerName)
+	raw := getDefaultListRawInput(rawContainerURLWithSAS.String())
+	raw.Properties = "VersionId"
+	raw.RunningTally = true
+
+	runListAndVerify(a, raw, func(err error) {
+		a.Nil(err)
+
+		// validate that the no transfers were scheduled
+		a.Nil(mockedRPC.transfers)
+
+		// check if info logs contain the correct version id for each blob
+		msg := mockedLcm.GatherAllLogs(mockedLcm.infoLog)
+		for i, m := range msg {
+			if i < 3 { // 0-2 will be blob names + version id
+				a.True(strings.Contains(m, blobsToInclude[i]))
+				a.True(strings.Contains(m, "VersionId: "+versions[blobsToInclude[i]]))
+			}
+			if i == 4 { // 4 will be file count
+				a.True(strings.Contains(m, "File count: 3"))
+			}
+			if i == 5 { // 5 will be file size
+				a.True(strings.Contains(m, "Total file size: 69.00 B"))
+			}
+		}
+	})
+
+}
+
+func TestListVersionsMultiVersions(t *testing.T) {
+	a := assert.New(t)
+	bsc := getSecondaryBlobServiceClient()
+	// set up the container with single blob with 2 versions
+	containerClient, containerName := createNewContainer(a, bsc)
+	defer deleteContainer(a, containerClient)
+
+	// testing how running tally will handle foo.txt vs foo/foo.txt, test/foo.txt
+	blobsToInclude := []string{"foo.txt", "foo/foo.txt", "test/foo.txt", "sub1/test/baz.txt"}
+	scenarioHelper{}.generateBlobsFromList(a, containerClient, blobsToInclude, blockBlobDefaultData)
+	a.NotNil(containerClient)
+
+	// make first two blobs have 1 additional version
+	blobsToVersion := []string{blobsToInclude[0], blobsToInclude[1]}
+	scenarioHelper{}.generateVersionsForBlobsFromList(a, containerClient, blobsToVersion)
+	a.NotNil(containerClient)
+
+	// confirm that container has 6 blobs (4 blobs, 2 versions)
+	pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
+		Include: container.ListBlobsInclude{Versions: true},
+	})
+	list, err := pager.NextPage(ctx)
+	a.NoError(err)
+	a.NotNil(list.Segment.BlobItems)
+	a.Equal(6, len(list.Segment.BlobItems))
+
+	var blobs []string
+	var versions []string
+	for _, item := range list.Segment.BlobItems {
+		blobs = append(blobs, *item.Name)
+		versions = append(versions, *item.VersionID)
+	}
+
+	// set up interceptor
+	mockedRPC := interceptor{}
+	Rpc = mockedRPC.intercept
+	mockedRPC.init()
+
+	mockedLcm := mockedLifecycleManager{infoLog: make(chan string, 50)}
+	mockedLcm.SetOutputFormat(common.EOutputFormat.Text()) // text format
+	glcm = &mockedLcm
+
+	// construct the raw input to simulate user input
+	rawContainerURLWithSAS := scenarioHelper{}.getSecondaryRawContainerURLWithSAS(a, containerName)
+	raw := getDefaultListRawInput(rawContainerURLWithSAS.String())
+	raw.Properties = "VersionId"
+	raw.RunningTally = true
+
+	runListAndVerify(a, raw, func(err error) {
+		a.Nil(err)
+
+		// validate that the no transfers were scheduled
+		a.Nil(mockedRPC.transfers)
+
+		// check if info logs contain the correct version id for each blob
+		msg := mockedLcm.GatherAllLogs(mockedLcm.infoLog)
+		for i, m := range msg {
+			if i < 6 { // 0-5 will be blob names + version id
+				a.True(contains(blobs, m, true))
+				a.True(contains(versions, m, false))
+			}
+			if i == 7 { // 7 will be file count
+				a.True(strings.Contains(m, "File count: 4"))
+			}
+			if i == 8 { // 8 will be file size
+				a.True(strings.Contains(m, "Total file size: 92.00 B"))
+			}
+		}
+	})
+
+}
+
+func contains(arr []string, msg string, isBlob bool) bool {
+	for _, a := range arr {
+		if isBlob {
+			if strings.HasPrefix(msg, a) {
+				return true
+			}
+		} else {
+			if strings.Contains(msg, a) {
+				return true
+			}
+		}
+	}
+	return false
+}