diff --git a/cmd/copy.go b/cmd/copy.go index 641fe4783..56ec99c66 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -81,6 +81,7 @@ type rawCopyCmdArgs struct { excludeRegex string includeFileAttributes string excludeFileAttributes string + excludeContainer string includeBefore string includeAfter string legacyInclude string // used only for warnings @@ -854,6 +855,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) { cooked.IncludePatterns = raw.parsePatterns(raw.include) cooked.ExcludePatterns = raw.parsePatterns(raw.exclude) cooked.ExcludePathPatterns = raw.parsePatterns(raw.excludePath) + cooked.excludeContainer = raw.parsePatterns(raw.excludeContainer) if (raw.includeFileAttributes != "" || raw.excludeFileAttributes != "") && fromTo.From() != common.ELocation.Local() { return cooked, errors.New("cannot check file attributes on remote objects") @@ -1097,6 +1099,7 @@ type CookedCopyCmdArgs struct { IncludePatterns []string ExcludePatterns []string ExcludePathPatterns []string + excludeContainer []string IncludeFileAttributes []string ExcludeFileAttributes []string IncludeBefore *time.Time @@ -1563,7 +1566,7 @@ func (cca *CookedCopyCmdArgs) processCopyJobPartOrders() (err error) { } if cca.FromTo.To() == common.ELocation.File() { - azureFileSpecificOptions = &common.FileClientOptions { + azureFileSpecificOptions = &common.FileClientOptions{ AllowTrailingDot: cca.trailingDot == common.ETrailingDotOption.Enable(), AllowSourceTrailingDot: cca.trailingDot == common.ETrailingDotOption.Enable() && cca.FromTo.From() == common.ELocation.File(), } @@ -2078,6 +2081,7 @@ func init() { cpCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "check-md5", common.DefaultHashValidationOption.String(), "Specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading. Available options: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing. (default 'FailIfDifferent')") cpCmd.PersistentFlags().StringVar(&raw.includeFileAttributes, "include-attributes", "", "(Windows only) Include files whose attributes match the attribute list. For example: A;S;R") cpCmd.PersistentFlags().StringVar(&raw.excludeFileAttributes, "exclude-attributes", "", "(Windows only) Exclude files whose attributes match the attribute list. For example: A;S;R") + cpCmd.PersistentFlags().StringVar(&raw.excludeContainer, "exclude-container", "", "Exclude these containers when transferring from Account to Account only.") cpCmd.PersistentFlags().BoolVar(&raw.CheckLength, "check-length", true, "Check the length of a file on the destination after the transfer. If there is a mismatch between source and destination, the transfer is marked as failed.") cpCmd.PersistentFlags().BoolVar(&raw.s2sPreserveProperties, "s2s-preserve-properties", true, "Preserve full properties during service to service copy. "+ "For AWS S3 and Azure File non-single file source, the list operation doesn't return full properties of objects and files. To preserve full properties, AzCopy needs to send one additional request per object or file.") diff --git a/cmd/copyEnumeratorInit.go b/cmd/copyEnumeratorInit.go index afec3f35e..f85031d68 100755 --- a/cmd/copyEnumeratorInit.go +++ b/cmd/copyEnumeratorInit.go @@ -65,14 +65,14 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde (cca.FromTo.From() == common.ELocation.File() && !cca.FromTo.To().IsRemote()) || // If it's a download, we still need LMT and MD5 from files. (cca.FromTo.From() == common.ELocation.File() && cca.FromTo.To().IsRemote() && (cca.s2sSourceChangeValidation || cca.IncludeAfter != nil || cca.IncludeBefore != nil)) || // If S2S from File to *, and sourceChangeValidation is enabled, we get properties so that we have LMTs. Likewise, if we are using includeAfter or includeBefore, which require LMTs. (cca.FromTo.From().IsRemote() && cca.FromTo.To().IsRemote() && cca.s2sPreserveProperties && !cca.s2sGetPropertiesInBackend) // If S2S and preserve properties AND get properties in backend is on, turn this off, as properties will be obtained in the backend. - jobPartOrder.S2SGetPropertiesInBackend = cca.s2sPreserveProperties && !getRemoteProperties && cca.s2sGetPropertiesInBackend // Infer GetProperties if GetPropertiesInBackend is enabled. + jobPartOrder.S2SGetPropertiesInBackend = cca.s2sPreserveProperties && !getRemoteProperties && cca.s2sGetPropertiesInBackend // Infer GetProperties if GetPropertiesInBackend is enabled. jobPartOrder.S2SSourceChangeValidation = cca.s2sSourceChangeValidation jobPartOrder.DestLengthValidation = cca.CheckLength jobPartOrder.S2SInvalidMetadataHandleOption = cca.s2sInvalidMetadataHandleOption jobPartOrder.S2SPreserveBlobTags = cca.S2sPreserveBlobTags dest := cca.FromTo.To() - traverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &srcCredInfo, cca.SymlinkHandling, cca.ListOfFilesChannel, cca.Recursive, getRemoteProperties, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, cca.S2sPreserveBlobTags, common.ESyncHashType.None(), cca.preservePermissions, azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, &dest) + traverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &srcCredInfo, cca.SymlinkHandling, cca.ListOfFilesChannel, cca.Recursive, getRemoteProperties, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, cca.S2sPreserveBlobTags, common.ESyncHashType.None(), cca.preservePermissions, azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, &dest, cca.excludeContainer) if err != nil { return nil, err @@ -343,7 +343,7 @@ func (cca *CookedCopyCmdArgs) isDestDirectory(dst common.ResourceString, ctx *co return false } - rt, err := InitResourceTraverser(dst, cca.FromTo.To(), ctx, &dstCredInfo, common.ESymlinkHandlingType.Skip(), nil, false, false, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), cca.preservePermissions, common.LogNone, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil) + rt, err := InitResourceTraverser(dst, cca.FromTo.To(), ctx, &dstCredInfo, common.ESymlinkHandlingType.Skip(), nil, false, false, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), cca.preservePermissions, common.LogNone, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil, cca.excludeContainer) if err != nil { return false @@ -485,7 +485,7 @@ func (cca *CookedCopyCmdArgs) createDstContainer(containerName string, dstWithSA } return err case common.ELocation.File(): - fsc, _:= sc.FileServiceClient() + fsc, _ := sc.FileServiceClient() sc := fsc.NewShareClient(containerName) _, err = sc.GetProperties(ctx, nil) diff --git a/cmd/list.go b/cmd/list.go index a8c131db9..80e609c71 100755 --- a/cmd/list.go +++ b/cmd/list.go @@ -42,7 +42,7 @@ type rawListCmdArgs struct { MachineReadable bool RunningTally bool MegaUnits bool - trailingDot string + trailingDot string } type validProperty string @@ -116,7 +116,7 @@ type cookedListCmdArgs struct { MachineReadable bool RunningTally bool MegaUnits bool - trailingDot common.TrailingDotOption + trailingDot common.TrailingDotOption } var raw rawListCmdArgs @@ -162,8 +162,8 @@ func init() { listContainerCmd.PersistentFlags().BoolVar(&raw.RunningTally, "running-tally", false, "Counts the total number of files and their sizes.") listContainerCmd.PersistentFlags().BoolVar(&raw.MegaUnits, "mega-units", false, "Displays units in orders of 1000, not 1024.") listContainerCmd.PersistentFlags().StringVar(&raw.Properties, "properties", "", "delimiter (;) separated values of properties required in list output.") - listContainerCmd.PersistentFlags().StringVar(&raw.trailingDot, "trailing-dot", "", "'Enable' by default to treat file share related operations in a safe manner. Available options: Enable, Disable. " + - "Choose 'Disable' to go back to legacy (potentially unsafe) treatment of trailing dot files where the file service will trim any trailing dots in paths. This can result in potential data corruption if the transfer contains two paths that differ only by a trailing dot (ex: mypath and mypath.). If this flag is set to 'Disable' and AzCopy encounters a trailing dot file, it will warn customers in the scanning log but will not attempt to abort the operation." + + listContainerCmd.PersistentFlags().StringVar(&raw.trailingDot, "trailing-dot", "", "'Enable' by default to treat file share related operations in a safe manner. Available options: Enable, Disable. "+ + "Choose 'Disable' to go back to legacy (potentially unsafe) treatment of trailing dot files where the file service will trim any trailing dots in paths. This can result in potential data corruption if the transfer contains two paths that differ only by a trailing dot (ex: mypath and mypath.). If this flag is set to 'Disable' and AzCopy encounters a trailing dot file, it will warn customers in the scanning log but will not attempt to abort the operation."+ "If the destination does not support trailing dot files (Windows or Blob Storage), AzCopy will fail if the trailing dot file is the root of the transfer and skip any trailing dot paths encountered during enumeration.") rootCmd.AddCommand(listContainerCmd) @@ -237,7 +237,7 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { } } - traverser, err := InitResourceTraverser(source, cooked.location, &ctx, &credentialInfo, common.ESymlinkHandlingType.Skip(), nil, true, true, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, nil, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), common.LogNone, common.CpkOptions{}, nil, false, cooked.trailingDot, nil) + traverser, err := InitResourceTraverser(source, cooked.location, &ctx, &credentialInfo, common.ESymlinkHandlingType.Skip(), nil, true, true, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, nil, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), common.LogNone, common.CpkOptions{}, nil, false, cooked.trailingDot, nil, nil) if err != nil { return fmt.Errorf("failed to initialize traverser: %s", err.Error()) diff --git a/cmd/removeEnumerator.go b/cmd/removeEnumerator.go index 349bb4a94..7ae69be9b 100755 --- a/cmd/removeEnumerator.go +++ b/cmd/removeEnumerator.go @@ -50,7 +50,7 @@ func newRemoveEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator, er ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) // Include-path is handled by ListOfFilesChannel. - sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo, common.ESymlinkHandlingType.Skip(), cca.ListOfFilesChannel, cca.Recursive, true, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil) + sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo, common.ESymlinkHandlingType.Skip(), cca.ListOfFilesChannel, cca.Recursive, true, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil, cca.excludeContainer) // report failure to create traverser if err != nil { @@ -145,7 +145,7 @@ func removeBfsResources(cca *CookedCopyCmdArgs) (err error) { ctx := context.WithValue(context.Background(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) sourceURL, _ := cca.Source.String() options := createClientOptions(common.AzcopyCurrentJobLogger) - + targetServiceClient, err := common.GetServiceClientForLocation(cca.FromTo.From(), sourceURL, cca.credentialInfo.CredentialType, cca.credentialInfo.OAuthTokenInfo.TokenCredential, &options, nil) if err != nil { return err diff --git a/cmd/setPropertiesEnumerator.go b/cmd/setPropertiesEnumerator.go index 9ec866bf1..75c41556a 100755 --- a/cmd/setPropertiesEnumerator.go +++ b/cmd/setPropertiesEnumerator.go @@ -46,7 +46,7 @@ func setPropertiesEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator } // Include-path is handled by ListOfFilesChannel. - sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo, common.ESymlinkHandlingType.Preserve(), cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil) + sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo, common.ESymlinkHandlingType.Preserve(), cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity, cca.CpkOptions, nil, cca.StripTopDir, cca.trailingDot, nil, cca.excludeContainer) // report failure to create traverser if err != nil { diff --git a/cmd/sync.go b/cmd/sync.go index 60a26251b..94e4477ed 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -90,7 +90,7 @@ type rawSyncCmdArgs struct { // Provided key name will be fetched from Azure Key Vault and will be used to encrypt the data cpkScopeInfo string // dry run mode bool - dryrun bool + dryrun bool trailingDot string } @@ -428,7 +428,7 @@ type cookedSyncCmdArgs struct { mirrorMode bool - dryrunMode bool + dryrunMode bool trailingDot common.TrailingDotOption } @@ -802,8 +802,8 @@ func init() { syncCmd.PersistentFlags().BoolVar(&raw.cpkInfo, "cpk-by-value", false, "Client provided key by name let clients making requests against Azure Blob storage an option to provide an encryption key on a per-request basis. Provided key and its hash will be fetched from environment variables") syncCmd.PersistentFlags().BoolVar(&raw.mirrorMode, "mirror-mode", false, "Disable last-modified-time based comparison and overwrites the conflicting files and blobs at the destination if this flag is set to true. Default is false") syncCmd.PersistentFlags().BoolVar(&raw.dryrun, "dry-run", false, "Prints the path of files that would be copied or removed by the sync command. This flag does not copy or remove the actual files.") - syncCmd.PersistentFlags().StringVar(&raw.trailingDot, "trailing-dot", "", "'Enable' by default to treat file share related operations in a safe manner. Available options: Enable, Disable. " + - "Choose 'Disable' to go back to legacy (potentially unsafe) treatment of trailing dot files where the file service will trim any trailing dots in paths. This can result in potential data corruption if the transfer contains two paths that differ only by a trailing dot (ex: mypath and mypath.). If this flag is set to 'Disable' and AzCopy encounters a trailing dot file, it will warn customers in the scanning log but will not attempt to abort the operation." + + syncCmd.PersistentFlags().StringVar(&raw.trailingDot, "trailing-dot", "", "'Enable' by default to treat file share related operations in a safe manner. Available options: Enable, Disable. "+ + "Choose 'Disable' to go back to legacy (potentially unsafe) treatment of trailing dot files where the file service will trim any trailing dots in paths. This can result in potential data corruption if the transfer contains two paths that differ only by a trailing dot (ex: mypath and mypath.). If this flag is set to 'Disable' and AzCopy encounters a trailing dot file, it will warn customers in the scanning log but will not attempt to abort the operation."+ "If the destination does not support trailing dot files (Windows or Blob Storage), AzCopy will fail if the trailing dot file is the root of the transfer and skip any trailing dot paths encountered during enumeration.") syncCmd.PersistentFlags().StringVar(&raw.compareHash, "compare-hash", "None", "Inform sync to rely on hashes as an alternative to LMT. Missing hashes at a remote source will throw an error. (None, MD5) Default: None") diff --git a/cmd/syncEnumerator.go b/cmd/syncEnumerator.go index 0291ebcc8..3faf7ba26 100644 --- a/cmd/syncEnumerator.go +++ b/cmd/syncEnumerator.go @@ -43,7 +43,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s return nil, err } - if cca.fromTo.IsS2S() && srcCredInfo.CredentialType != common.ECredentialType.Anonymous() { + if cca.fromTo.IsS2S() && srcCredInfo.CredentialType != common.ECredentialType.Anonymous() { if srcCredInfo.CredentialType.IsAzureOAuth() && cca.fromTo.To().CanForwardOAuthTokens() { // no-op, this is OK } else if srcCredInfo.CredentialType == common.ECredentialType.GoogleAppCredentials() || srcCredInfo.CredentialType == common.ECredentialType.S3AccessKey() || srcCredInfo.CredentialType == common.ECredentialType.S3PublicBucket() { @@ -64,7 +64,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s if entityType == common.EEntityType.File() { atomic.AddUint64(&cca.atomicSourceFilesScanned, 1) } - }, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity, cca.cpkOptions, nil, false, cca.trailingDot, &dest) + }, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity, cca.cpkOptions, nil, false, cca.trailingDot, &dest, nil) if err != nil { return nil, err @@ -85,7 +85,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s if entityType == common.EEntityType.File() { atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1) } - }, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity, cca.cpkOptions, nil, false, cca.trailingDot, nil) + }, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity, cca.cpkOptions, nil, false, cca.trailingDot, nil, nil) if err != nil { return nil, err } @@ -139,7 +139,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s if cca.trailingDot == common.ETrailingDotOption.Enable() && !cca.fromTo.BothSupportTrailingDot() { cca.trailingDot = common.ETrailingDotOption.Disable() } - + copyJobTemplate := &common.CopyJobPartOrderRequest{ JobID: cca.jobID, CommandString: cca.commandString, @@ -175,13 +175,12 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s }, } - options := createClientOptions(common.AzcopyCurrentJobLogger) - - // Create Source Client. + + // Create Source Client. var azureFileSpecificOptions any if cca.fromTo.From() == common.ELocation.File() { - azureFileSpecificOptions = &common.FileClientOptions { + azureFileSpecificOptions = &common.FileClientOptions{ AllowTrailingDot: cca.trailingDot == common.ETrailingDotOption.Enable(), } } @@ -201,12 +200,12 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s // Create Destination client if cca.fromTo.To() == common.ELocation.File() { - azureFileSpecificOptions = &common.FileClientOptions { - AllowTrailingDot: cca.trailingDot == common.ETrailingDotOption.Enable(), + azureFileSpecificOptions = &common.FileClientOptions{ + AllowTrailingDot: cca.trailingDot == common.ETrailingDotOption.Enable(), AllowSourceTrailingDot: (cca.trailingDot == common.ETrailingDotOption.Enable() && cca.fromTo.To() == common.ELocation.File()), } } - + dstURL, _ := cca.destination.String() copyJobTemplate.DstServiceClient, err = common.GetServiceClientForLocation( cca.fromTo.To(), diff --git a/cmd/zc_enumerator.go b/cmd/zc_enumerator.go index e129c2ba7..5cf7e8bf2 100644 --- a/cmd/zc_enumerator.go +++ b/cmd/zc_enumerator.go @@ -334,7 +334,7 @@ type enumerationCounterFunc func(entityType common.EntityType) // errorOnDirWOutRecursive is used by copy. // If errorChannel is non-nil, all errors encountered during enumeration will be conveyed through this channel. // To avoid slowdowns, use a buffered channel of enough capacity. -func InitResourceTraverser(resource common.ResourceString, location common.Location, ctx *context.Context, credential *common.CredentialInfo, symlinkHandling common.SymlinkHandlingType, listOfFilesChannel chan string, recursive, getProperties, includeDirectoryStubs bool, permanentDeleteOption common.PermanentDeleteOption, incrementEnumerationCounter enumerationCounterFunc, listOfVersionIds chan string, s2sPreserveBlobTags bool, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption, logLevel common.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo, stripTopDir bool, trailingDot common.TrailingDotOption, destination *common.Location) (ResourceTraverser, error) { +func InitResourceTraverser(resource common.ResourceString, location common.Location, ctx *context.Context, credential *common.CredentialInfo, symlinkHandling common.SymlinkHandlingType, listOfFilesChannel chan string, recursive, getProperties, includeDirectoryStubs bool, permanentDeleteOption common.PermanentDeleteOption, incrementEnumerationCounter enumerationCounterFunc, listOfVersionIds chan string, s2sPreserveBlobTags bool, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption, logLevel common.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo, stripTopDir bool, trailingDot common.TrailingDotOption, destination *common.Location, excludeContainerNames []string) (ResourceTraverser, error) { var output ResourceTraverser var includeDeleted bool @@ -453,7 +453,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat if !recursive { return nil, errors.New(accountTraversalInherentlyRecursiveError) } - output = newBlobAccountTraverser(bsc, containerName, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, preservePermissions, false) + output = newBlobAccountTraverser(bsc, containerName, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, preservePermissions, false, excludeContainerNames) } else if listOfVersionIds != nil { output = newBlobVersionsTraverser(r, bsc, *ctx, includeDirectoryStubs, incrementEnumerationCounter, listOfVersionIds, cpkOptions) } else { @@ -540,7 +540,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat if !recursive { return nil, errors.New(accountTraversalInherentlyRecursiveError) } - output = newBlobAccountTraverser(bsc, containerName, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, preservePermissions, true) + output = newBlobAccountTraverser(bsc, containerName, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, preservePermissions, true, excludeContainerNames) } else if listOfVersionIds != nil { output = newBlobVersionsTraverser(r, bsc, *ctx, includeDirectoryStubs, incrementEnumerationCounter, listOfVersionIds, cpkOptions) } else { diff --git a/cmd/zc_filter.go b/cmd/zc_filter.go index f5d48f8cb..a329d4d32 100644 --- a/cmd/zc_filter.go +++ b/cmd/zc_filter.go @@ -59,6 +59,39 @@ func (f *excludeBlobTypeFilter) DoesPass(object StoredObject) bool { return false } +// excludeContainerFilter filters out container names that must be excluded +type excludeContainerFilter struct { + containerNamesList map[string]bool +} + +func (s *excludeContainerFilter) DoesSupportThisOS() (msg string, supported bool) { + return "", true +} + +func (s *excludeContainerFilter) AppliesOnlyToFiles() bool { + return false // excludeContainerFilter is related to container names, not related to files +} + +func (s *excludeContainerFilter) DoesPass(storedObject StoredObject) bool { + if len(s.containerNamesList) == 0 { + return true + } + + if _, exists := s.containerNamesList[storedObject.ContainerName]; exists { + return false + } + return true +} + +func buildExcludeContainerFilter(containerNames []string) []ObjectFilter { + excludeContainerSet := make(map[string]bool) + for _, name := range containerNames { + excludeContainerSet[name] = true + } + + return append(make([]ObjectFilter, 0), &excludeContainerFilter{containerNamesList: excludeContainerSet}) +} + type excludeFilter struct { pattern string targetsPath bool diff --git a/cmd/zc_traverser_blob_account.go b/cmd/zc_traverser_blob_account.go index 98f6eb091..458c7c009 100644 --- a/cmd/zc_traverser_blob_account.go +++ b/cmd/zc_traverser_blob_account.go @@ -25,6 +25,7 @@ import ( "fmt" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" "github.com/Azure/azure-storage-azcopy/v10/common" + "strings" ) // Enumerates an entire blob account, looking into each matching container as it goes @@ -44,6 +45,8 @@ type blobAccountTraverser struct { preservePermissions common.PreservePermissionsOption isDFS bool + + excludeContainerName []ObjectFilter } func (t *blobAccountTraverser) IsDirectory(_ bool) (bool, error) { @@ -51,42 +54,65 @@ func (t *blobAccountTraverser) IsDirectory(_ bool) (bool, error) { } func (t *blobAccountTraverser) listContainers() ([]string, error) { + cachedContainers, _, err := t.getListContainers() + return cachedContainers, err +} + +func (t *blobAccountTraverser) getListContainers() ([]string, []string, error) { + var skippedContainers []string // a nil list also returns 0 - if len(t.cachedContainers) == 0 { + if len(t.cachedContainers) == 0 || len(t.excludeContainerName) > 0 { cList := make([]string, 0) pager := t.serviceClient.NewListContainersPager(nil) for pager.More() { resp, err := pager.NextPage(t.ctx) if err != nil { - return nil, err + return nil, nil, err } for _, v := range resp.ContainerItems { - // Match a pattern for the container name and the container name only. - if t.containerPattern != "" { - if ok, err := containerNameMatchesPattern(*v.Name, t.containerPattern); err != nil { - // Break if the pattern is invalid - return nil, err - } else if !ok { - // Ignore the container if it doesn't match the pattern. - continue + // a nil list also returns 0 + if len(t.cachedContainers) == 0 { + // Match a pattern for the container name and the container name only. + if t.containerPattern != "" { + if ok, err := containerNameMatchesPattern(*v.Name, t.containerPattern); err != nil { + // Break if the pattern is invalid + return nil, nil, err + } else if !ok { + // Ignore the container if it doesn't match the pattern. + continue + } } } - cList = append(cList, *v.Name) + // get a list of containers that are not excluded + if len(t.excludeContainerName) > 0 { + so := StoredObject{ContainerName: *v.Name} + for _, f := range t.excludeContainerName { + if !f.DoesPass(so) { + // Ignore the container if the container name should be excluded + skippedContainers = append(skippedContainers, *v.Name) + continue + } else { + cList = append(cList, *v.Name) + } + } + } else { + cList = append(cList, *v.Name) + } } } - t.cachedContainers = cList - - return cList, nil - } else { - return t.cachedContainers, nil } + + return t.cachedContainers, skippedContainers, nil } func (t *blobAccountTraverser) Traverse(preprocessor objectMorpher, processor objectProcessor, filters []ObjectFilter) error { // listContainers will return the cached container list if containers have already been listed by this traverser. - cList, err := t.listContainers() + cList, skippedContainers, err := t.getListContainers() + if len(skippedContainers) > 0 { + glcm.Info("Skipped container(s): " + strings.Join(skippedContainers, ", ")) + } if err != nil { return err @@ -109,7 +135,7 @@ func (t *blobAccountTraverser) Traverse(preprocessor objectMorpher, processor ob return nil } -func newBlobAccountTraverser(serviceClient *service.Client, container string, ctx context.Context, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, preservePermissions common.PreservePermissionsOption, isDFS bool) (t *blobAccountTraverser) { +func newBlobAccountTraverser(serviceClient *service.Client, container string, ctx context.Context, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, preservePermissions common.PreservePermissionsOption, isDFS bool, containerNames []string) (t *blobAccountTraverser) { t = &blobAccountTraverser{ ctx: ctx, incrementEnumerationCounter: incrementEnumerationCounter, @@ -119,7 +145,8 @@ func newBlobAccountTraverser(serviceClient *service.Client, container string, ct s2sPreserveSourceTags: s2sPreserveSourceTags, cpkOptions: cpkOptions, preservePermissions: preservePermissions, - isDFS: isDFS, + isDFS: isDFS, + excludeContainerName: buildExcludeContainerFilter(containerNames), } return diff --git a/cmd/zc_traverser_list.go b/cmd/zc_traverser_list.go index c17d2fd4f..fa96f898d 100755 --- a/cmd/zc_traverser_list.go +++ b/cmd/zc_traverser_list.go @@ -107,7 +107,7 @@ func newListTraverser(parent common.ResourceString, parentType common.Location, } // Construct a traverser that goes through the child - traverser, err := InitResourceTraverser(source, parentType, ctx, credential, handleSymlinks, nil, recursive, getProperties, includeDirectoryStubs, common.EPermanentDeleteOption.None(), incrementEnumerationCounter, nil, s2sPreserveBlobTags, syncHashType, preservePermissions, logLevel, cpkOptions, nil, false, trailingDot, destination) + traverser, err := InitResourceTraverser(source, parentType, ctx, credential, handleSymlinks, nil, recursive, getProperties, includeDirectoryStubs, common.EPermanentDeleteOption.None(), incrementEnumerationCounter, nil, s2sPreserveBlobTags, syncHashType, preservePermissions, logLevel, cpkOptions, nil, false, trailingDot, destination, nil) if err != nil { return nil, err } diff --git a/cmd/zt_copy_account_account_test.go b/cmd/zt_copy_account_account_test.go new file mode 100644 index 000000000..37bb3d084 --- /dev/null +++ b/cmd/zt_copy_account_account_test.go @@ -0,0 +1,142 @@ +// Copyright © Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/stretchr/testify/assert" + "strings" + "testing" +) + +// test copy +func TestExcludeContainerFlagCopy(t *testing.T) { + a := assert.New(t) + srcBSC := scenarioHelper{}.getBlobServiceClientWithSAS(a) + dstBSC := scenarioHelper{}.getSecondaryBlobServiceClientWithSAS(a) + + // set up 3 containers with blobs on source account + containerNames := []string{"foo", "bar", "baz"} + containersToIgnore := []string{containerNames[0], containerNames[1]} + blobNames := []string{"stuff-1", "stuff-2"} + var containerClients []*container.Client + + for _, name := range containerNames { + // create container client + cc := srcBSC.NewContainerClient(name) + _, err := cc.Create(ctx, nil) + a.NoError(err) + + // create blobs + scenarioHelper{}.generateBlobsFromList(a, cc, blobNames, blockBlobDefaultData) + + // append to array of container clients + containerClients = append(containerClients, cc) + } + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + raw := getDefaultCopyRawInput(srcBSC.URL(), dstBSC.URL()) + raw.recursive = true + raw.excludeContainer = strings.Join(containersToIgnore, ";") + + runCopyAndVerify(a, raw, func(err error) { + a.Nil(err) + + // validate that each transfer is not of the excluded container names + for _, transfer := range mockedRPC.transfers { + a.NotNil(transfer) + for _, excludeName := range containersToIgnore { + a.False(strings.Contains(transfer.Source, excludeName)) + a.False(strings.Contains(transfer.Destination, excludeName)) + } + } + }) + + // deleting test containers from source acc + for _, container := range containerNames { + cc := srcBSC.NewContainerClient(container) + deleteContainer(a, cc) + } + + // deleting test containers from dst acc + deleteContainer(a, dstBSC.NewContainerClient(containerNames[2])) + +} + +func TestExcludeContainerFlagCopyNegative(t *testing.T) { + a := assert.New(t) + srcBSC := scenarioHelper{}.getBlobServiceClientWithSAS(a) + dstBSC := scenarioHelper{}.getSecondaryBlobServiceClientWithSAS(a) + + // set up 2 containers with blobs on source account + containerNames := []string{"hello", "world"} + // ignore a container name that doesn't actually exist, AzCopy will continue as normal + containersToIgnore := []string{"xxx"} + blobNames := []string{"stuff-1", "stuff-2"} + var containerClients []*container.Client + + for _, name := range containerNames { + // create container client + cc := srcBSC.NewContainerClient(name) + _, err := cc.Create(ctx, nil) + a.NoError(err) + + // create blobs + scenarioHelper{}.generateBlobsFromList(a, cc, blobNames, blockBlobDefaultData) + + // append to array of container clients + containerClients = append(containerClients, cc) + } + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + raw := getDefaultCopyRawInput(srcBSC.URL(), dstBSC.URL()) + raw.recursive = true + raw.excludeContainer = strings.Join(containersToIgnore, ";") + + runCopyAndVerify(a, raw, func(err error) { + a.Nil(err) + + // validate that each transfer is not of the excluded container names + for _, transfer := range mockedRPC.transfers { + a.NotNil(transfer) + for _, excludeName := range containersToIgnore { + a.False(strings.Contains(transfer.Source, excludeName)) + a.False(strings.Contains(transfer.Destination, excludeName)) + } + } + }) + + // deleting test containers from source acc + for i, _ := range containerNames { + deleteContainer(a, srcBSC.NewContainerClient(containerNames[i])) + deleteContainer(a, dstBSC.NewContainerClient(containerNames[i])) + } +} diff --git a/cmd/zt_copy_blob_download_test.go b/cmd/zt_copy_blob_download_test.go index 2c4cacfd9..6189e46c9 100644 --- a/cmd/zt_copy_blob_download_test.go +++ b/cmd/zt_copy_blob_download_test.go @@ -178,7 +178,7 @@ func TestDownloadAccount(t *testing.T) { // Traverse the account ahead of time and determine the relative paths for testing. relPaths := make([]string, 0) // Use a map for easy lookup - blobTraverser := newBlobAccountTraverser(rawBSC, "", ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None(), false) + blobTraverser := newBlobAccountTraverser(rawBSC, "", ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None(), false, nil) processor := func(object StoredObject) error { // Skip non-file types _, ok := object.Metadata[common.POSIXSymlinkMeta] @@ -233,7 +233,7 @@ func TestDownloadAccountWildcard(t *testing.T) { // Traverse the account ahead of time and determine the relative paths for testing. relPaths := make([]string, 0) // Use a map for easy lookup - blobTraverser := newBlobAccountTraverser(rawBSC, container, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None(), false) + blobTraverser := newBlobAccountTraverser(rawBSC, container, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None(), false, nil) processor := func(object StoredObject) error { // Skip non-file types _, ok := object.Metadata[common.POSIXSymlinkMeta] diff --git a/cmd/zt_generic_service_traverser_test.go b/cmd/zt_generic_service_traverser_test.go index 503d62028..ebe239bba 100644 --- a/cmd/zt_generic_service_traverser_test.go +++ b/cmd/zt_generic_service_traverser_test.go @@ -105,7 +105,7 @@ func TestServiceTraverserWithManyObjects(t *testing.T) { // construct a blob account traverser rawBSU := scenarioHelper{}.getBlobServiceClientWithSAS(a) - blobAccountTraverser := newBlobAccountTraverser(rawBSU, "", ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None(), false) + blobAccountTraverser := newBlobAccountTraverser(rawBSU, "", ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None(), false, nil) // invoke the blob account traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -273,7 +273,7 @@ func TestServiceTraverserWithWildcards(t *testing.T) { // construct a blob account traverser rawBSU := scenarioHelper{}.getBlobServiceClientWithSAS(a) container := "objectmatch*" // set the container name to contain a wildcard - blobAccountTraverser := newBlobAccountTraverser(rawBSU, container, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None(), false) + blobAccountTraverser := newBlobAccountTraverser(rawBSU, container, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None(), false, nil) // invoke the blob account traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -350,4 +350,4 @@ func TestServiceTraverserWithWildcards(t *testing.T) { a.True(cnamePresent) a.Equal(storedObject.name, correspondingLocalFile.name) } -} \ No newline at end of file +} diff --git a/cmd/zt_generic_traverser_test.go b/cmd/zt_generic_traverser_test.go index 87249121a..7f976a038 100644 --- a/cmd/zt_generic_traverser_test.go +++ b/cmd/zt_generic_traverser_test.go @@ -103,6 +103,7 @@ func TestLocalWildcardOverlap(t *testing.T) { true, common.ETrailingDotOption.Enable(), nil, + nil, ) a.Nil(err) @@ -115,7 +116,7 @@ func TestLocalWildcardOverlap(t *testing.T) { a.Nil(err) a.Equal(map[string]bool{ - "test.txt": true, + "test.txt": true, "tes*t.txt": true, }, seenFiles) } @@ -991,4 +992,4 @@ func TestSerialAndParallelBlobTraverser(t *testing.T) { a.Equal(correspondingFile.md5, storedObject.md5) } } -} \ No newline at end of file +} diff --git a/cmd/zt_scenario_helpers_for_test.go b/cmd/zt_scenario_helpers_for_test.go index 3924c263d..4b834f818 100644 --- a/cmd/zt_scenario_helpers_for_test.go +++ b/cmd/zt_scenario_helpers_for_test.go @@ -658,6 +658,14 @@ func (scenarioHelper) getBlobServiceClientWithSAS(a *assert.Assertions) *blobser return getBlobServiceClientWithSAS(a, credential) } +func (scenarioHelper) getSecondaryBlobServiceClientWithSAS(a *assert.Assertions) *blobservice.Client { + accountName, accountKey := getSecondaryAccountAndKey() + credential, err := blob.NewSharedKeyCredential(accountName, accountKey) + a.Nil(err) + + return getBlobServiceClientWithSAS(a, credential) +} + func (scenarioHelper) getBlobServiceClientWithSASFromURL(a *assert.Assertions, rawURL string) *blobservice.Client { blobURLParts, err := blob.ParseURL(rawURL) a.Nil(err)