Merge pull request #1684 from Azure/dev

AzCopy Release 10.14.0

Author: zezha-msft

ChangeLog.md

@@ -1,6 +1,32 @@
 
 # Change Log
 
+## Version 10.14.0
+
+### New features
+1. Feature to [permanently delete](https://docs.microsoft.com/en-us/rest/api/storageservices/delete-blob#remarks) soft-deleted
+   snapshots/versions of the blobs has been added (preview). `--permanent-delete=none/snapshots/version/snapshotsandversions`.
+2. Feature to preserve properties and ACLs when copying to Azure file share root directory.
+3. Pin all APIs to use the default service version `2020-04-08` and let users decide the service version via
+   `AZCOPY_DEFAULT_SERVICE_API_VERSION` environment variable. Previously, few APIs were not respecting the `AZCOPY_DEFAULT_SERVICE_API_VERSION` environment variable.
+
+### Bug fixes
+1. Fixed issue in which AzCopy failed to copy to classic blob container with `preserve blob access tier`.
+2. Fixed [issue 1630](https://github.com/Azure/azure-storage-azcopy/issues/1630) : AzCopy created extra empty
+   directories at destination while performing S2S transfer from one ADLS Gen2 account to another ADLS Gen2 account.
+3. Changed the way AzCopy was using to obtain and set ACLs to ensure accuracy.
+4. Clarify error message for `azcopy sync` when source or destination cannot be detected.
+5. Report error when client provided key(CPK) encryption is applied to DFS endpoint.
+6. Fixed [issue 1596](https://github.com/Azure/azure-storage-azcopy/issues/1596) : AzCopy failed to transfer files 
+   (with '/.' in their path) from AWS S3 to Azure blob storage.
+7. Fixed [issue 1474](https://github.com/Azure/azure-storage-azcopy/issues/1474) : AzCopy panicked when trying to re-create an already open plan file.
+8. Improved handling of Auth error against single file.
+9. Fixed [issue 1640](https://github.com/Azure/azure-storage-azcopy/issues/1640) : Recursive copy from GCS bucket to Azure container failed 
+   with `FileIgnored` error when using `--exclude-path`.
+10. Fixed [issue 1655](https://github.com/Azure/azure-storage-azcopy/issues/1655) : AzCopy panicked when using `--include-before` flag.
+11. Fixed [issue 1609](https://github.com/Azure/azure-storage-azcopy/issues/1609) : `blockid` converted to lower case in AzCopy logs.
+12. Fixed [issue 1643](https://github.com/Azure/azure-storage-azcopy/issues/1643), [issue 1661](https://github.com/Azure/azure-storage-azcopy/issues/1661) : Updated Golang version to `1.16.10` to fix security vulnerabilities in Golang packages. 
+
 ## Version 10.13.0
 
 ### New features

azbfs/parsing_urls.go

@@ -80,6 +80,12 @@ func NewBfsURLParts(u url.URL) BfsURLParts {
 	return up
 }
 
+func GetSasQueryParams(sas string) SASQueryParameters {
+	paramsMap, _ := url.ParseQuery(sas)
+	SAS := newSASQueryParameters(paramsMap, true)
+	return SAS
+}
+
 // URL returns a URL object whose fields are initialized from the BfsURLParts fields.
 func (up BfsURLParts) URL() url.URL {
 	path := ""

azbfs/url_directory.go

@@ -174,12 +174,17 @@ func (d DirectoryURL) Rename(ctx context.Context, options RenameDirectoryOptions
 		fileSystemName = &d.filesystem
 	}
 
-	renameSource := "/" + d.filesystem + "/" + d.pathParameter
-
 	urlParts := NewBfsURLParts(d.directoryClient.URL())
 	urlParts.FileSystemName = *fileSystemName
 	urlParts.DirectoryOrFilePath = options.DestinationPath
+	// ensure we use our source's SAS token in the x-ms-rename-source header
+	renameSource := "/" + d.filesystem + "/" + d.pathParameter + "?" + urlParts.SAS.Encode()
 
+	// if we're changing our source SAS to a new SAS in the rename
+	// in the case the user wants to have limited permissions per directory: sas1 for directory1 and sas2 for directory2
+	if options.DestinationSas != nil && *options.DestinationSas != "" {
+		urlParts.SAS = GetSasQueryParams(*options.DestinationSas)
+	}
 	destinationDirectoryURL := NewDirectoryURL(urlParts.URL(), d.directoryClient.Pipeline())
 
 	_, err := destinationDirectoryURL.directoryClient.Create(ctx, *fileSystemName, options.DestinationPath, PathResourceNone, nil, PathRenameModeLegacy,

azbfs/url_file.go

@@ -29,9 +29,9 @@ type BlobFSHTTPHeaders struct {
 
 // BlobFSAccessControl represents the set of custom headers available for defining access conditions for the content.
 type BlobFSAccessControl struct {
-	Owner 		string
-	Group 		string
-	ACL   		string // Combining ACL & Permissions = invalid for SetAccessControl.
+	Owner       string
+	Group       string
+	ACL         string // Combining ACL & Permissions = invalid for SetAccessControl.
 	Permissions string
 }
 
@@ -72,18 +72,18 @@ func (f FileURL) GetParentDir() (DirectoryURL, error) {
 
 // Create creates a new file or replaces a file. Note that this method only initializes the file.
 // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create.
-func (f FileURL) Create(ctx context.Context, headers BlobFSHTTPHeaders) (*PathCreateResponse, error) {
-	return f.CreateWithOptions(ctx, CreateFileOptions{Headers: headers})
+func (f FileURL) Create(ctx context.Context, headers BlobFSHTTPHeaders, permissions BlobFSAccessControl) (*PathCreateResponse, error) {
+	return f.CreateWithOptions(ctx, CreateFileOptions{Headers: headers}, permissions)
 }
 
 // Create creates a new file or replaces a file. Note that this method only initializes the file.
 // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create.
-func (f FileURL) CreateWithOptions(ctx context.Context, options CreateFileOptions) (*PathCreateResponse, error) {
+func (f FileURL) CreateWithOptions(ctx context.Context, options CreateFileOptions, permissions BlobFSAccessControl) (*PathCreateResponse, error) {
 	return f.fileClient.Create(ctx, f.fileSystemName, f.path, PathResourceFile,
 		nil, PathRenameModeNone, nil, nil, nil, nil,
 		&options.Headers.CacheControl, &options.Headers.ContentType, &options.Headers.ContentEncoding,
 		&options.Headers.ContentLanguage, &options.Headers.ContentDisposition, nil, nil, nil,
-		buildMetadataString(options.Metadata), nil, nil,
+		buildMetadataString(options.Metadata), &permissions.Permissions, nil,
 		nil, nil, nil, nil, nil,
 		nil, nil, nil, nil, nil,
 		nil)
@@ -224,12 +224,17 @@ func (f FileURL) Rename(ctx context.Context, options RenameFileOptions) (FileURL
 		fileSystemName = &f.fileSystemName
 	}
 
-	renameSource := "/" + f.fileSystemName + "/" + f.path
-
 	urlParts := NewBfsURLParts(f.fileClient.URL())
 	urlParts.FileSystemName = *fileSystemName
 	urlParts.DirectoryOrFilePath = options.DestinationPath
+	// ensure we use our source's SAS token in the x-ms-rename-source header
+	renameSource := "/" + f.fileSystemName + "/" + f.path + "?" + urlParts.SAS.Encode()
 
+	// if we're changing our source SAS to a new SAS in the rename
+	// in the case the user wants to have limited permissions per directory: sas1 for file1 and sas2 for file2
+	if options.DestinationSas != nil && *options.DestinationSas != "" {
+		urlParts.SAS = GetSasQueryParams(*options.DestinationSas)
+	}
 	destinationFileURL := NewFileURL(urlParts.URL(), f.fileClient.Pipeline())
 
 	_, err := destinationFileURL.fileClient.Create(ctx, *fileSystemName, options.DestinationPath, PathResourceNone, nil, PathRenameModeLegacy,
@@ -279,4 +284,4 @@ func (f FileURL) SetAccessControl(ctx context.Context, permissions BlobFSAccessC
 		nil, nil, &permissions.Owner, &permissions.Group, perms, acl,
 		nil, nil, nil, nil, &overrideHttpVerb,
 		nil, nil, nil, nil)
-}
+}

azbfs/zc_directoryRequestOptions.go

@@ -17,4 +17,6 @@ type RenameDirectoryOptions struct {
 	DestinationFileSystem *string
 	// The destination path for the directory.
 	DestinationPath string
-}
+	// The new SAS for a destination Directory
+	DestinationSas *string
+}

azbfs/zc_fileRequestOptions.go

@@ -17,4 +17,6 @@ type RenameFileOptions struct {
 	DestinationFileSystem *string
 	// The destination path for the file.
 	DestinationPath string
-}
+	// The new SAS for a destination file
+	DestinationSas *string
+}

azbfs/zt_test.go

@@ -141,7 +141,7 @@ func createNewFileFromFileSystem(c *chk.C, fileSystem azbfs.FileSystemURL) (file
 
 	file, name = getFileURLFromDirectory(c, dir)
 
-	cResp, err := file.Create(ctx, azbfs.BlobFSHTTPHeaders{})
+	cResp, err := file.Create(ctx, azbfs.BlobFSHTTPHeaders{}, azbfs.BlobFSAccessControl{})
 	c.Assert(err, chk.IsNil)
 	c.Assert(cResp.StatusCode(), chk.Equals, 201)
 

azbfs/zt_url_directory_test.go

@@ -2,7 +2,11 @@ package azbfs_test
 
 import (
 	"context"
+	"fmt"
 	"net/http"
+	"net/url"
+	"strings"
+	"time"
 
 	"github.com/Azure/azure-storage-azcopy/v10/azbfs"
 	chk "gopkg.in/check.v1"
@@ -127,7 +131,7 @@ func (dus *DirectoryUrlSuite) TestCreateDirectoryAndFiles(c *chk.C) {
 
 	// Create fileUrl from directoryUrl and create file inside the directory
 	fileUrl, _ := getFileURLFromDirectory(c, dirUrl)
-	fresp, err := fileUrl.Create(context.Background(), azbfs.BlobFSHTTPHeaders{})
+	fresp, err := fileUrl.Create(context.Background(), azbfs.BlobFSHTTPHeaders{}, azbfs.BlobFSAccessControl{})
 	defer delFile(c, fileUrl)
 
 	c.Assert(err, chk.IsNil)
@@ -237,7 +241,7 @@ func (dus *DirectoryUrlSuite) TestDirectoryStructure(c *chk.C) {
 
 	// Create a file inside directory
 	fileUrl, _ := getFileURLFromDirectory(c, dirUrl)
-	fresp, err := fileUrl.Create(context.Background(), azbfs.BlobFSHTTPHeaders{})
+	fresp, err := fileUrl.Create(context.Background(), azbfs.BlobFSHTTPHeaders{}, azbfs.BlobFSAccessControl{})
 	defer delFile(c, fileUrl)
 
 	c.Assert(err, chk.IsNil)
@@ -250,7 +254,7 @@ func (dus *DirectoryUrlSuite) TestDirectoryStructure(c *chk.C) {
 
 	// create a file inside the sub-dir created above
 	subDirfileUrl, _ := getFileURLFromDirectory(c, subDirUrl)
-	fresp, err = subDirfileUrl.Create(context.Background(), azbfs.BlobFSHTTPHeaders{})
+	fresp, err = subDirfileUrl.Create(context.Background(), azbfs.BlobFSHTTPHeaders{}, azbfs.BlobFSAccessControl{})
 	defer delFile(c, subDirfileUrl)
 
 	c.Assert(err, chk.IsNil)
@@ -292,7 +296,7 @@ func (dus *DirectoryUrlSuite) TestListDirectoryWithSpaces(c *chk.C) {
 
 	// Create a file inside directory
 	fileUrl, _ := getFileURLFromDirectory(c, dirUrl)
-	_, err = fileUrl.Create(context.Background(), azbfs.BlobFSHTTPHeaders{})
+	_, err = fileUrl.Create(context.Background(), azbfs.BlobFSHTTPHeaders{}, azbfs.BlobFSAccessControl{})
 	defer delFile(c, fileUrl)
 
 	// list the directory created above.
@@ -394,7 +398,7 @@ func (dus *DirectoryUrlSuite) TestSetACL(c *chk.C) {
 
 	// Create a file
 	fileUrl := dirURL.NewFileURL("foo.bar")
-	_, err = fileUrl.Create(ctx, azbfs.BlobFSHTTPHeaders{})
+	_, err = fileUrl.Create(ctx, azbfs.BlobFSHTTPHeaders{}, azbfs.BlobFSAccessControl{})
 	c.Assert(err, chk.IsNil)
 
 	// Grab it's default ACLs
@@ -416,3 +420,98 @@ func (dus *DirectoryUrlSuite) TestSetACL(c *chk.C) {
 
 	// Don't bother testing the root ACLs, since it calls into the directoryclient
 }
+
+func (s *FileURLSuite) TestRenameDirectoryWithSas(c *chk.C) {
+	name, key := getAccountAndKey()
+	credential := azbfs.NewSharedKeyCredential(name, key)
+	sasQueryParams, err := azbfs.AccountSASSignatureValues{
+		Protocol:      azbfs.SASProtocolHTTPS,
+		ExpiryTime:    time.Now().Add(48 * time.Hour),
+		Permissions:   azbfs.AccountSASPermissions{Read: true, List: true, Write: true, Delete: true, Add: true, Create: true, Update: true, Process: true}.String(),
+		Services:      azbfs.AccountSASServices{File: true, Blob: true, Queue: true}.String(),
+		ResourceTypes: azbfs.AccountSASResourceTypes{Service: true, Container: true, Object: true}.String(),
+	}.NewSASQueryParameters(credential)
+	c.Assert(err, chk.IsNil)
+
+	qp := sasQueryParams.Encode()
+	rawURL := fmt.Sprintf("https://%s.dfs.core.windows.net/?%s",
+		credential.AccountName(), qp)
+	fullURL, err := url.Parse(rawURL)
+	c.Assert(err, chk.IsNil)
+
+	fsu := azbfs.NewServiceURL(*fullURL, azbfs.NewPipeline(azbfs.NewAnonymousCredential(), azbfs.PipelineOptions{}))
+
+	fileSystemURL, _ := createNewFileSystem(c, fsu)
+	defer delFileSystem(c, fileSystemURL)
+
+	dirURL, dirName := createNewDirectoryFromFileSystem(c, fileSystemURL)
+	dirRename := dirName + "rename"
+
+	renamedDirURL, err := dirURL.Rename(context.Background(), azbfs.RenameDirectoryOptions{DestinationPath: dirRename})
+	c.Assert(renamedDirURL, chk.NotNil)
+	c.Assert(err, chk.IsNil)
+
+	// Check that the old directory does not exist
+	getPropertiesResp, err := dirURL.GetProperties(context.Background())
+	c.Assert(err, chk.NotNil) // TODO: I want to check the status code is 404 but not sure how since the resp is nil
+	c.Assert(getPropertiesResp, chk.IsNil)
+
+	// Check that the renamed directory does exist
+	getPropertiesResp, err = renamedDirURL.GetProperties(context.Background())
+	c.Assert(getPropertiesResp.StatusCode(), chk.Equals, http.StatusOK)
+	c.Assert(err, chk.IsNil)
+}
+
+func (s *FileURLSuite) TestRenameDirectoryWithDestinationSas(c *chk.C) {
+	name, key := getAccountAndKey()
+	credential := azbfs.NewSharedKeyCredential(name, key)
+	sourceSasQueryParams, err := azbfs.AccountSASSignatureValues{
+		Protocol:      azbfs.SASProtocolHTTPS,
+		ExpiryTime:    time.Now().Add(48 * time.Hour),
+		Permissions:   azbfs.AccountSASPermissions{Read: true, List: true, Write: true, Delete: true, Add: true, Create: true, Update: true, Process: true}.String(),
+		Services:      azbfs.AccountSASServices{File: true, Blob: true, Queue: true}.String(),
+		ResourceTypes: azbfs.AccountSASResourceTypes{Service: true, Container: true, Object: true}.String(),
+	}.NewSASQueryParameters(credential)
+	c.Assert(err, chk.IsNil)
+	destinationSasQueryParams, err := azbfs.AccountSASSignatureValues{
+		Protocol:      azbfs.SASProtocolHTTPS,
+		ExpiryTime:    time.Now().Add(24 * time.Hour),
+		Permissions:   azbfs.AccountSASPermissions{Read: true, Write: true, Delete: true, Add: true, Create: true, Update: true, Process: true}.String(),
+		Services:      azbfs.AccountSASServices{File: true, Blob: true}.String(),
+		ResourceTypes: azbfs.AccountSASResourceTypes{Service: true, Container: true, Object: true}.String(),
+	}.NewSASQueryParameters(credential)
+	c.Assert(err, chk.IsNil)
+
+	sourceQp := sourceSasQueryParams.Encode()
+	destQp := destinationSasQueryParams.Encode()
+	rawURL := fmt.Sprintf("https://%s.dfs.core.windows.net/?%s",
+		credential.AccountName(), sourceQp)
+	fullURL, err := url.Parse(rawURL)
+	c.Assert(err, chk.IsNil)
+
+	fsu := azbfs.NewServiceURL(*fullURL, azbfs.NewPipeline(azbfs.NewAnonymousCredential(), azbfs.PipelineOptions{}))
+
+	fileSystemURL, _ := createNewFileSystem(c, fsu)
+	defer delFileSystem(c, fileSystemURL)
+
+	dirURL, dirName := createNewDirectoryFromFileSystem(c, fileSystemURL)
+	dirRename := dirName + "rename"
+
+	renamedDirURL, err := dirURL.Rename(
+		context.Background(), azbfs.RenameDirectoryOptions{DestinationPath: dirRename, DestinationSas: &destQp})
+	c.Assert(renamedDirURL, chk.NotNil)
+	c.Assert(err, chk.IsNil)
+	found := strings.Contains(renamedDirURL.String(), destQp)
+	// make sure the correct SAS is used
+	c.Assert(found, chk.Equals, true)
+
+	// Check that the old directory does not exist
+	getPropertiesResp, err := dirURL.GetProperties(context.Background())
+	c.Assert(err, chk.NotNil) // TODO: I want to check the status code is 404 but not sure how since the resp is nil
+	c.Assert(getPropertiesResp, chk.IsNil)
+
+	// Check that the renamed directory does exist
+	getPropertiesResp, err = renamedDirURL.GetProperties(context.Background())
+	c.Assert(getPropertiesResp.StatusCode(), chk.Equals, http.StatusOK)
+	c.Assert(err, chk.IsNil)
+}