From f5245c0bc20d6df0d9d7b3a6982af6ed9a86123d Mon Sep 17 00:00:00 2001 From: Pranav Malik Date: Wed, 17 Jan 2024 12:47:00 +0530 Subject: [PATCH 1/5] Ignoring the 404 error --- ste/xfer-deleteBlobFS.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/ste/xfer-deleteBlobFS.go b/ste/xfer-deleteBlobFS.go index d89a9b8cd..e34425a2b 100644 --- a/ste/xfer-deleteBlobFS.go +++ b/ste/xfer-deleteBlobFS.go @@ -1,8 +1,10 @@ package ste import ( + "errors" "fmt" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake" @@ -14,6 +16,7 @@ import ( ) var logBlobFSDeleteWarnOnce = &sync.Once{} + const blobFSDeleteWarning = "Displayed file count will be either 1 or based upon list-of-files entries, and thus inaccurate, as deletes are performed recursively service-side." func DeleteHNSResource(jptm IJobPartTransferMgr, pacer pacer) { @@ -51,7 +54,16 @@ func doDeleteHNSResource(jptm IJobPartTransferMgr) { transferDone := func(err error) { status := common.ETransferStatus.Success() if err != nil { - status = common.ETransferStatus.Failed() + var respErr *azcore.ResponseError + if errors.As(err, &respErr) { + if respErr.StatusCode == http.StatusNotFound { + // if the delete failed with err 404, i.e resource not found, then mark the transfer as success. + status = common.ETransferStatus.Success() + } else { + // in all other cases, make the transfer as failed + status = common.ETransferStatus.Failed() + } + } } if status == common.ETransferStatus.Failed() { @@ -70,7 +82,7 @@ func doDeleteHNSResource(jptm IJobPartTransferMgr) { transferDone(err) return } - + c := s.NewFileSystemClient(jptm.Info().SrcContainer) // Deleting a filesystem @@ -102,4 +114,4 @@ func doDeleteHNSResource(jptm IJobPartTransferMgr) { _, err := directoryClient.Delete(recursiveContext, nil) transferDone(err) } -} \ No newline at end of file +} From 4b7396cf1c1c286bc38806c9f83ac2d36533f300 Mon Sep 17 00:00:00 2001 From: Pranav Malik Date: Thu, 18 Jan 2024 11:22:12 +0530 Subject: [PATCH 2/5] adding 403 support --- ste/xfer-deleteBlobFS.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ste/xfer-deleteBlobFS.go b/ste/xfer-deleteBlobFS.go index e34425a2b..9692633b1 100644 --- a/ste/xfer-deleteBlobFS.go +++ b/ste/xfer-deleteBlobFS.go @@ -59,6 +59,13 @@ func doDeleteHNSResource(jptm IJobPartTransferMgr) { if respErr.StatusCode == http.StatusNotFound { // if the delete failed with err 404, i.e resource not found, then mark the transfer as success. status = common.ETransferStatus.Success() + } + // If the status code was 403, it means there was an authentication error and we exit. + // User can resume the job if completely ordered with a new sas. + if respErr.StatusCode == http.StatusForbidden { + errMsg := fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()) + jptm.Log(common.LogError, errMsg) + common.GetLifecycleMgr().Error(errMsg) } else { // in all other cases, make the transfer as failed status = common.ETransferStatus.Failed() From fcfb0cbe21123f090d05f7a20cd967dd77d6f240 Mon Sep 17 00:00:00 2001 From: Pranav Malik Date: Fri, 26 Jan 2024 12:57:03 +0530 Subject: [PATCH 3/5] adding testing support --- ste/sender-appendBlob_test.go | 60 +++++++- ste/sender_blockBlob_test.go | 5 +- ste/sourceInfoProvider_md5_test.go | 33 ++--- ste/testJobPartTransferManager_test.go | 189 ++++++++++++------------- 4 files changed, 169 insertions(+), 118 deletions(-) diff --git a/ste/sender-appendBlob_test.go b/ste/sender-appendBlob_test.go index 13b8a830b..3b3d548e8 100644 --- a/ste/sender-appendBlob_test.go +++ b/ste/sender-appendBlob_test.go @@ -3,6 +3,10 @@ package ste import ( "context" "fmt" + "net/http" + "testing" + "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" @@ -13,9 +17,6 @@ import ( blobservice "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" "github.com/Azure/azure-storage-azcopy/v10/common" "github.com/stretchr/testify/assert" - "net/http" - "testing" - "time" ) type appendErrorInjectionPolicy struct { @@ -81,7 +82,7 @@ func Test500FollowedBy412Logic(t *testing.T) { }), fromTo: common.EFromTo.BlobBlob(), } - blobSIP, err := newBlobSourceInfoProvider(jptm) + blobSIP, err := newBlobSourceInfoProvider(&jptm) a.Nil(err) injectionPolicy := &appendErrorInjectionPolicy{timedOut: false} @@ -92,7 +93,7 @@ func Test500FollowedBy412Logic(t *testing.T) { }, }) a.Nil(err) - base := appendBlobSenderBase{jptm: jptm, destAppendBlobClient: destClient, sip: blobSIP} + base := appendBlobSenderBase{jptm: &jptm, destAppendBlobClient: destClient, sip: blobSIP} // Get MD5 range within service calculation offset := int64(0) @@ -108,3 +109,52 @@ func Test500FollowedBy412Logic(t *testing.T) { a.Nil(err) a.Empty(errString) } + +// This function tests the scenario where we return a transfer success even when we receive a 404 response, indicating a resource not found error. +// In this test, we create a container on an HNS enabled account but do not create any blob. This is done to simulate the 404 scenario when attempting to delete a non-existent blob. +// The deletion operation won't find the blob to delete, resulting in a 404 error, and thus returning a transfer success. +func Test404DeleteSuccessLogic(t *testing.T) { + a := assert.New(t) + + // Setup source and destination + accountName, accountKey := getAccountAndKey() + rawURL := fmt.Sprintf("https://%s.dfs.core.windows.net/", accountName) + + credential, err := blob.NewSharedKeyCredential(accountName, accountKey) + a.Nil(err) + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: NewAzcopyHTTPClient(0), + }}) + a.Nil(err) + + cName := generateContainerName() + cc := client.NewContainerClient(cName) + _, err = cc.Create(context.Background(), nil) + a.Nil(err) + defer cc.Delete(context.Background(), nil) + + // Generating the name for a blob without actually creating it. + sourceName := generateBlobName() + + sasURL, err := cc.NewBlobClient(sourceName).GetSASURL( + blobsas.BlobPermissions{Read: true}, + time.Now().Add(1*time.Hour), + nil) + a.Nil(err) + + jptm := &testJobPartTransferManager{ + info: to.Ptr(TransferInfo{ + Source: sasURL, + SrcContainer: cName, + SrcFilePath: sourceName, + }), + fromTo: common.EFromTo.BlobFSTrash(), + } + jptm.SetStatus(common.ETransferStatus.Started()) + doDeleteHNSResource(jptm) + + a.Nil(err) + a.Equal(jptm.status, common.ETransferStatus.Success()) + +} diff --git a/ste/sender_blockBlob_test.go b/ste/sender_blockBlob_test.go index 6c27c90d1..7d5925628 100644 --- a/ste/sender_blockBlob_test.go +++ b/ste/sender_blockBlob_test.go @@ -21,13 +21,14 @@ package ste import ( + "strings" + "testing" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" "github.com/Azure/azure-storage-azcopy/v10/common" "github.com/stretchr/testify/assert" - "strings" - "testing" ) func TestGetVerifiedChunkParams(t *testing.T) { diff --git a/ste/sourceInfoProvider_md5_test.go b/ste/sourceInfoProvider_md5_test.go index e4bfef27e..99d7d0b61 100644 --- a/ste/sourceInfoProvider_md5_test.go +++ b/ste/sourceInfoProvider_md5_test.go @@ -22,11 +22,20 @@ package ste import ( "bytes" - gcpUtils "cloud.google.com/go/storage" "context" "crypto/md5" "errors" "fmt" + "hash/crc64" + "io" + "math/rand" + "os" + "runtime" + "strings" + "testing" + "time" + + gcpUtils "cloud.google.com/go/storage" "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" @@ -38,14 +47,6 @@ import ( "github.com/Azure/azure-storage-azcopy/v10/common" "github.com/minio/minio-go" "github.com/stretchr/testify/assert" - "hash/crc64" - "io" - "math/rand" - "os" - "runtime" - "strings" - "testing" - "time" ) // This file covers testing of the GetMD5() method in the various sourceInfoProvider implementations @@ -174,7 +175,7 @@ func TestBenchmark(t *testing.T) { info: nil, fromTo: common.EFromTo.BenchmarkBlob(), } - benchSIP, err := newBenchmarkSourceInfoProvider(jptm) + benchSIP, err := newBenchmarkSourceInfoProvider(&jptm) a.Nil(err) _, err = benchSIP.GetMD5(0, 1) @@ -220,7 +221,7 @@ func TestBlockBlob(t *testing.T) { }), fromTo: common.EFromTo.BlobBlob(), } - blobSIP, err := newBlobSourceInfoProvider(jptm) + blobSIP, err := newBlobSourceInfoProvider(&jptm) a.Nil(err) // Get MD5 range within service calculation @@ -288,7 +289,7 @@ func TestShareFile(t *testing.T) { }), fromTo: common.EFromTo.FileBlob(), } - fileSIP, err := newFileSourceInfoProvider(jptm) + fileSIP, err := newFileSourceInfoProvider(&jptm) a.Nil(err) // Get MD5 range within service calculation @@ -357,7 +358,7 @@ func TestShareDirectory(t *testing.T) { }), fromTo: common.EFromTo.FileBlob(), } - fileSIP, err := newFileSourceInfoProvider(jptm) + fileSIP, err := newFileSourceInfoProvider(&jptm) a.Nil(err) _, err = fileSIP.GetMD5(0, 1) @@ -398,7 +399,7 @@ func TestGCP(t *testing.T) { }), fromTo: common.EFromTo.GCPBlob(), } - gcpSIP, err := newGCPSourceInfoProvider(jptm) + gcpSIP, err := newGCPSourceInfoProvider(&jptm) a.Nil(err) // Get MD5 range within service calculation @@ -448,7 +449,7 @@ func TestLocal(t *testing.T) { }), fromTo: common.EFromTo.LocalBlob(), } - localSIP, err := newLocalSourceInfoProvider(jptm) + localSIP, err := newLocalSourceInfoProvider(&jptm) a.Nil(err) // Get MD5 range within service calculation @@ -504,7 +505,7 @@ func TestS3(t *testing.T) { }), fromTo: common.EFromTo.S3Blob(), } - s3SIP, err := newS3SourceInfoProvider(jptm) + s3SIP, err := newS3SourceInfoProvider(&jptm) a.Nil(err) // Get MD5 range within service calculation diff --git a/ste/testJobPartTransferManager_test.go b/ste/testJobPartTransferManager_test.go index 20d2de3e6..61b675db2 100644 --- a/ste/testJobPartTransferManager_test.go +++ b/ste/testJobPartTransferManager_test.go @@ -22,32 +22,34 @@ package ste import ( "context" + "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-storage-azcopy/v10/common" - "time" ) -var _ IJobPartTransferMgr = testJobPartTransferManager{} +var _ IJobPartTransferMgr = &testJobPartTransferManager{} type testJobPartTransferManager struct { info *TransferInfo fromTo common.FromTo jobPartMgr jobPartMgr ctx context.Context + status common.TransferStatus } -func (t testJobPartTransferManager) DeleteDestinationFileIfNecessary() bool { +func (t *testJobPartTransferManager) DeleteDestinationFileIfNecessary() bool { return t.jobPartMgr.DeleteDestinationFileIfNecessary() } -func (t testJobPartTransferManager) Info() *TransferInfo { +func (t *testJobPartTransferManager) Info() *TransferInfo { return t.info } -func (t testJobPartTransferManager) SrcServiceClient() *common.ServiceClient { +func (t *testJobPartTransferManager) SrcServiceClient() *common.ServiceClient { options := t.S2SSourceClientOptions() var azureFileSpecificOptions any if t.fromTo.From() == common.ELocation.File() { @@ -66,7 +68,7 @@ func (t testJobPartTransferManager) SrcServiceClient() *common.ServiceClient { return client } -func (t testJobPartTransferManager) DstServiceClient() *common.ServiceClient { +func (t *testJobPartTransferManager) DstServiceClient() *common.ServiceClient { options := t.ClientOptions() var azureFileSpecificOptions any if t.fromTo.To() == common.ELocation.File() { @@ -86,186 +88,187 @@ func (t testJobPartTransferManager) DstServiceClient() *common.ServiceClient { return client } -func (t testJobPartTransferManager) SourceTrailingDot() *common.TrailingDotOption { +func (t *testJobPartTransferManager) SourceTrailingDot() *common.TrailingDotOption { if (t.fromTo.IsS2S() || t.fromTo.IsDownload()) && (t.fromTo.From() == common.ELocation.File()) { return to.Ptr(common.ETrailingDotOption.Enable()) } return nil } -func (t testJobPartTransferManager) TrailingDot() *common.TrailingDotOption { +func (t *testJobPartTransferManager) TrailingDot() *common.TrailingDotOption { return to.Ptr(common.ETrailingDotOption.Enable()) } -func (t testJobPartTransferManager) From() *common.Location { +func (t *testJobPartTransferManager) From() *common.Location { return to.Ptr(t.fromTo.From()) } -func (t testJobPartTransferManager) FromTo() common.FromTo { +func (t *testJobPartTransferManager) FromTo() common.FromTo { return t.fromTo } -func (t testJobPartTransferManager) ResourceDstData(dataFileToXfer []byte) (headers common.ResourceHTTPHeaders, metadata common.Metadata, blobTags common.BlobTags, cpkOptions common.CpkOptions) { +func (t *testJobPartTransferManager) ResourceDstData(dataFileToXfer []byte) (headers common.ResourceHTTPHeaders, metadata common.Metadata, blobTags common.BlobTags, cpkOptions common.CpkOptions) { panic("implement me") } -func (t testJobPartTransferManager) LastModifiedTime() time.Time { +func (t *testJobPartTransferManager) LastModifiedTime() time.Time { panic("implement me") } -func (t testJobPartTransferManager) PreserveLastModifiedTime() (time.Time, bool) { +func (t *testJobPartTransferManager) PreserveLastModifiedTime() (time.Time, bool) { panic("implement me") } -func (t testJobPartTransferManager) ShouldPutMd5() bool { +func (t *testJobPartTransferManager) ShouldPutMd5() bool { panic("implement me") } -func (t testJobPartTransferManager) MD5ValidationOption() common.HashValidationOption { +func (t *testJobPartTransferManager) MD5ValidationOption() common.HashValidationOption { panic("implement me") } -func (t testJobPartTransferManager) BlobTypeOverride() common.BlobType { +func (t *testJobPartTransferManager) BlobTypeOverride() common.BlobType { panic("implement me") } -func (t testJobPartTransferManager) BlobTiers() (blockBlobTier common.BlockBlobTier, pageBlobTier common.PageBlobTier) { +func (t *testJobPartTransferManager) BlobTiers() (blockBlobTier common.BlockBlobTier, pageBlobTier common.PageBlobTier) { panic("implement me") } -func (t testJobPartTransferManager) JobHasLowFileCount() bool { +func (t *testJobPartTransferManager) JobHasLowFileCount() bool { panic("implement me") } -func (t testJobPartTransferManager) Context() context.Context { +func (t *testJobPartTransferManager) Context() context.Context { return context.Background() } -func (t testJobPartTransferManager) SlicePool() common.ByteSlicePooler { +func (t *testJobPartTransferManager) SlicePool() common.ByteSlicePooler { panic("implement me") } -func (t testJobPartTransferManager) CacheLimiter() common.CacheLimiter { +func (t *testJobPartTransferManager) CacheLimiter() common.CacheLimiter { panic("implement me") } -func (t testJobPartTransferManager) WaitUntilLockDestination(ctx context.Context) error { +func (t *testJobPartTransferManager) WaitUntilLockDestination(ctx context.Context) error { panic("implement me") } -func (t testJobPartTransferManager) EnsureDestinationUnlocked() { +func (t *testJobPartTransferManager) EnsureDestinationUnlocked() { panic("implement me") } -func (t testJobPartTransferManager) HoldsDestinationLock() bool { +func (t *testJobPartTransferManager) HoldsDestinationLock() bool { panic("implement me") } -func (t testJobPartTransferManager) StartJobXfer() { +func (t *testJobPartTransferManager) StartJobXfer() { panic("implement me") } -func (t testJobPartTransferManager) GetOverwriteOption() common.OverwriteOption { +func (t *testJobPartTransferManager) GetOverwriteOption() common.OverwriteOption { panic("implement me") } -func (t testJobPartTransferManager) GetForceIfReadOnly() bool { +func (t *testJobPartTransferManager) GetForceIfReadOnly() bool { panic("implement me") } -func (t testJobPartTransferManager) ShouldDecompress() bool { +func (t *testJobPartTransferManager) ShouldDecompress() bool { panic("implement me") } -func (t testJobPartTransferManager) GetSourceCompressionType() (common.CompressionType, error) { +func (t *testJobPartTransferManager) GetSourceCompressionType() (common.CompressionType, error) { panic("implement me") } -func (t testJobPartTransferManager) ReportChunkDone(id common.ChunkID) (lastChunk bool, chunksDone uint32) { +func (t *testJobPartTransferManager) ReportChunkDone(id common.ChunkID) (lastChunk bool, chunksDone uint32) { panic("implement me") } -func (t testJobPartTransferManager) TransferStatusIgnoringCancellation() common.TransferStatus { +func (t *testJobPartTransferManager) TransferStatusIgnoringCancellation() common.TransferStatus { panic("implement me") } -func (t testJobPartTransferManager) SetStatus(status common.TransferStatus) { - panic("implement me") +func (t *testJobPartTransferManager) SetStatus(status common.TransferStatus) { + t.status = status } -func (t testJobPartTransferManager) SetErrorCode(errorCode int32) { +func (t *testJobPartTransferManager) SetErrorCode(errorCode int32) { panic("implement me") } -func (t testJobPartTransferManager) SetNumberOfChunks(numChunks uint32) { +func (t *testJobPartTransferManager) SetNumberOfChunks(numChunks uint32) { panic("implement me") } -func (t testJobPartTransferManager) SetActionAfterLastChunk(f func()) { +func (t *testJobPartTransferManager) SetActionAfterLastChunk(f func()) { panic("implement me") } -func (t testJobPartTransferManager) ReportTransferDone() uint32 { - panic("implement me") +func (t *testJobPartTransferManager) ReportTransferDone() uint32 { + // return value is the no of transfer's done for this job part. + return 1 } -func (t testJobPartTransferManager) RescheduleTransfer() { +func (t *testJobPartTransferManager) RescheduleTransfer() { panic("implement me") } -func (t testJobPartTransferManager) ScheduleChunks(chunkFunc chunkFunc) { +func (t *testJobPartTransferManager) ScheduleChunks(chunkFunc chunkFunc) { panic("implement me") } -func (t testJobPartTransferManager) SetDestinationIsModified() { +func (t *testJobPartTransferManager) SetDestinationIsModified() { panic("implement me") } -func (t testJobPartTransferManager) Cancel() { +func (t *testJobPartTransferManager) Cancel() { panic("implement me") } -func (t testJobPartTransferManager) WasCanceled() bool { +func (t *testJobPartTransferManager) WasCanceled() bool { panic("implement me") } -func (t testJobPartTransferManager) IsLive() bool { +func (t *testJobPartTransferManager) IsLive() bool { panic("implement me") } -func (t testJobPartTransferManager) IsDeadBeforeStart() bool { +func (t *testJobPartTransferManager) IsDeadBeforeStart() bool { panic("implement me") } -func (t testJobPartTransferManager) IsDeadInflight() bool { +func (t *testJobPartTransferManager) IsDeadInflight() bool { panic("implement me") } -func (t testJobPartTransferManager) OccupyAConnection() { +func (t *testJobPartTransferManager) OccupyAConnection() { panic("implement me") } -func (t testJobPartTransferManager) ReleaseAConnection() { +func (t *testJobPartTransferManager) ReleaseAConnection() { panic("implement me") } -func (t testJobPartTransferManager) CredentialInfo() common.CredentialInfo { +func (t *testJobPartTransferManager) CredentialInfo() common.CredentialInfo { panic("implement me") } -func (t testJobPartTransferManager) ClientOptions() azcore.ClientOptions { +func (t *testJobPartTransferManager) ClientOptions() azcore.ClientOptions { panic("implement me") } -func (t testJobPartTransferManager) S2SSourceCredentialInfo() common.CredentialInfo { +func (t *testJobPartTransferManager) S2SSourceCredentialInfo() common.CredentialInfo { return common.CredentialInfo{CredentialType: common.ECredentialType.Anonymous()} } -func (t testJobPartTransferManager) GetS2SSourceTokenCredential(ctx context.Context) (token *string, err error) { +func (t *testJobPartTransferManager) GetS2SSourceTokenCredential(ctx context.Context) (token *string, err error) { panic("implement me") } -func (t testJobPartTransferManager) S2SSourceClientOptions() azcore.ClientOptions { +func (t *testJobPartTransferManager) S2SSourceClientOptions() azcore.ClientOptions { retryOptions := policy.RetryOptions{ MaxRetries: UploadMaxTries, TryTimeout: UploadTryTimeout, @@ -290,154 +293,150 @@ func (t testJobPartTransferManager) S2SSourceClientOptions() azcore.ClientOption return NewClientOptions(retryOptions, telemetryOptions, httpClient, nil, LogOptions{}, nil) } -func (t testJobPartTransferManager) CredentialOpOptions() *common.CredentialOpOptions { +func (t *testJobPartTransferManager) CredentialOpOptions() *common.CredentialOpOptions { return nil } -func (t testJobPartTransferManager) FailActiveUpload(where string, err error) { +func (t *testJobPartTransferManager) FailActiveUpload(where string, err error) { panic("implement me") } -func (t testJobPartTransferManager) FailActiveDownload(where string, err error) { +func (t *testJobPartTransferManager) FailActiveDownload(where string, err error) { panic("implement me") } -func (t testJobPartTransferManager) FailActiveUploadWithStatus(where string, err error, failureStatus common.TransferStatus) { +func (t *testJobPartTransferManager) FailActiveUploadWithStatus(where string, err error, failureStatus common.TransferStatus) { panic("implement me") } -func (t testJobPartTransferManager) FailActiveDownloadWithStatus(where string, err error, failureStatus common.TransferStatus) { +func (t *testJobPartTransferManager) FailActiveDownloadWithStatus(where string, err error, failureStatus common.TransferStatus) { panic("implement me") } -func (t testJobPartTransferManager) FailActiveS2SCopy(where string, err error) { +func (t *testJobPartTransferManager) FailActiveS2SCopy(where string, err error) { panic("implement me") } -func (t testJobPartTransferManager) FailActiveS2SCopyWithStatus(where string, err error, failureStatus common.TransferStatus) { +func (t *testJobPartTransferManager) FailActiveS2SCopyWithStatus(where string, err error, failureStatus common.TransferStatus) { panic("implement me") } -func (t testJobPartTransferManager) FailActiveSend(where string, err error) { +func (t *testJobPartTransferManager) FailActiveSend(where string, err error) { panic("implement me") } -func (t testJobPartTransferManager) FailActiveSendWithStatus(where string, err error, failureStatus common.TransferStatus) { +func (t *testJobPartTransferManager) FailActiveSendWithStatus(where string, err error, failureStatus common.TransferStatus) { panic("implement me") } -func (t testJobPartTransferManager) LogUploadError(source, destination, errorMsg string, status int) { +func (t *testJobPartTransferManager) LogUploadError(source, destination, errorMsg string, status int) { panic("implement me") } -func (t testJobPartTransferManager) LogDownloadError(source, destination, errorMsg string, status int) { +func (t *testJobPartTransferManager) LogDownloadError(source, destination, errorMsg string, status int) { panic("implement me") } -func (t testJobPartTransferManager) LogS2SCopyError(source, destination, errorMsg string, status int) { +func (t *testJobPartTransferManager) LogS2SCopyError(source, destination, errorMsg string, status int) { panic("implement me") } -func (t testJobPartTransferManager) LogSendError(source, destination, errorMsg string, status int) { +func (t *testJobPartTransferManager) LogSendError(source, destination, errorMsg string, status int) { panic("implement me") } -func (t testJobPartTransferManager) LogError(resource, context string, err error) { - panic("implement me") -} +func (t *testJobPartTransferManager) LogError(_, _ string, _ error) {} -func (t testJobPartTransferManager) LogTransferInfo(level common.LogLevel, source, destination, msg string) { - panic("implement me") -} +func (t *testJobPartTransferManager) LogTransferInfo(_ common.LogLevel, _, _, _ string) {} -func (t testJobPartTransferManager) LogTransferStart(source, destination, description string) { +func (t *testJobPartTransferManager) LogTransferStart(source, destination, description string) { panic("implement me") } -func (t testJobPartTransferManager) LogChunkStatus(id common.ChunkID, reason common.WaitReason) { +func (t *testJobPartTransferManager) LogChunkStatus(id common.ChunkID, reason common.WaitReason) { panic("implement me") } -func (t testJobPartTransferManager) ChunkStatusLogger() common.ChunkStatusLogger { +func (t *testJobPartTransferManager) ChunkStatusLogger() common.ChunkStatusLogger { panic("implement me") } -func (t testJobPartTransferManager) LogAtLevelForCurrentTransfer(level common.LogLevel, msg string) { +func (t *testJobPartTransferManager) LogAtLevelForCurrentTransfer(level common.LogLevel, msg string) { panic("implement me") } -func (t testJobPartTransferManager) GetOverwritePrompter() *overwritePrompter { +func (t *testJobPartTransferManager) GetOverwritePrompter() *overwritePrompter { panic("implement me") } -func (t testJobPartTransferManager) GetFolderCreationTracker() FolderCreationTracker { +func (t *testJobPartTransferManager) GetFolderCreationTracker() FolderCreationTracker { panic("implement me") } -func (t testJobPartTransferManager) ShouldLog(level common.LogLevel) bool { +func (t *testJobPartTransferManager) ShouldLog(level common.LogLevel) bool { return false } -func (t testJobPartTransferManager) Log(level common.LogLevel, msg string) { +func (t *testJobPartTransferManager) Log(level common.LogLevel, msg string) { } -func (t testJobPartTransferManager) Panic(err error) { +func (t *testJobPartTransferManager) Panic(err error) { panic("implement me") } -func (t testJobPartTransferManager) DeleteSnapshotsOption() common.DeleteSnapshotsOption { +func (t *testJobPartTransferManager) DeleteSnapshotsOption() common.DeleteSnapshotsOption { panic("implement me") } -func (t testJobPartTransferManager) PermanentDeleteOption() common.PermanentDeleteOption { +func (t *testJobPartTransferManager) PermanentDeleteOption() common.PermanentDeleteOption { panic("implement me") } -func (t testJobPartTransferManager) SecurityInfoPersistenceManager() *securityInfoPersistenceManager { +func (t *testJobPartTransferManager) SecurityInfoPersistenceManager() *securityInfoPersistenceManager { panic("implement me") } -func (t testJobPartTransferManager) FolderDeletionManager() common.FolderDeletionManager { +func (t *testJobPartTransferManager) FolderDeletionManager() common.FolderDeletionManager { panic("implement me") } -func (t testJobPartTransferManager) GetDestinationRoot() string { +func (t *testJobPartTransferManager) GetDestinationRoot() string { panic("implement me") } -func (t testJobPartTransferManager) ShouldInferContentType() bool { +func (t *testJobPartTransferManager) ShouldInferContentType() bool { fromTo := t.FromTo() return fromTo.From() == common.ELocation.Local() } -func (t testJobPartTransferManager) CpkInfo() *blob.CPKInfo { +func (t *testJobPartTransferManager) CpkInfo() *blob.CPKInfo { return nil } -func (t testJobPartTransferManager) CpkScopeInfo() *blob.CPKScopeInfo { +func (t *testJobPartTransferManager) CpkScopeInfo() *blob.CPKScopeInfo { return nil } -func (t testJobPartTransferManager) IsSourceEncrypted() bool { +func (t *testJobPartTransferManager) IsSourceEncrypted() bool { panic("implement me") } -func (t testJobPartTransferManager) PropertiesToTransfer() common.SetPropertiesFlags { +func (t *testJobPartTransferManager) PropertiesToTransfer() common.SetPropertiesFlags { panic("implement me") } -func (t testJobPartTransferManager) ResetSourceSize() { +func (t *testJobPartTransferManager) ResetSourceSize() { panic("implement me") } -func (t testJobPartTransferManager) SuccessfulBytesTransferred() int64 { +func (t *testJobPartTransferManager) SuccessfulBytesTransferred() int64 { panic("implement me") } -func (t testJobPartTransferManager) TransferIndex() (partNum, transferIndex uint32) { +func (t *testJobPartTransferManager) TransferIndex() (partNum, transferIndex uint32) { panic("implement me") } -func (t testJobPartTransferManager) RestartedTransfer() bool { +func (t *testJobPartTransferManager) RestartedTransfer() bool { return false } From d82e671615f0952bd5babdbfd098a010864ffa33 Mon Sep 17 00:00:00 2001 From: Pranav Malik Date: Mon, 29 Jan 2024 11:42:36 +0530 Subject: [PATCH 4/5] moving test to new file --- ste/sender-appendBlob_test.go | 49 ------------------------ ste/sender-deleteHNSBlob_test.go | 64 ++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 49 deletions(-) create mode 100644 ste/sender-deleteHNSBlob_test.go diff --git a/ste/sender-appendBlob_test.go b/ste/sender-appendBlob_test.go index 3b3d548e8..a231c614f 100644 --- a/ste/sender-appendBlob_test.go +++ b/ste/sender-appendBlob_test.go @@ -109,52 +109,3 @@ func Test500FollowedBy412Logic(t *testing.T) { a.Nil(err) a.Empty(errString) } - -// This function tests the scenario where we return a transfer success even when we receive a 404 response, indicating a resource not found error. -// In this test, we create a container on an HNS enabled account but do not create any blob. This is done to simulate the 404 scenario when attempting to delete a non-existent blob. -// The deletion operation won't find the blob to delete, resulting in a 404 error, and thus returning a transfer success. -func Test404DeleteSuccessLogic(t *testing.T) { - a := assert.New(t) - - // Setup source and destination - accountName, accountKey := getAccountAndKey() - rawURL := fmt.Sprintf("https://%s.dfs.core.windows.net/", accountName) - - credential, err := blob.NewSharedKeyCredential(accountName, accountKey) - a.Nil(err) - client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, &blobservice.ClientOptions{ - ClientOptions: azcore.ClientOptions{ - Transport: NewAzcopyHTTPClient(0), - }}) - a.Nil(err) - - cName := generateContainerName() - cc := client.NewContainerClient(cName) - _, err = cc.Create(context.Background(), nil) - a.Nil(err) - defer cc.Delete(context.Background(), nil) - - // Generating the name for a blob without actually creating it. - sourceName := generateBlobName() - - sasURL, err := cc.NewBlobClient(sourceName).GetSASURL( - blobsas.BlobPermissions{Read: true}, - time.Now().Add(1*time.Hour), - nil) - a.Nil(err) - - jptm := &testJobPartTransferManager{ - info: to.Ptr(TransferInfo{ - Source: sasURL, - SrcContainer: cName, - SrcFilePath: sourceName, - }), - fromTo: common.EFromTo.BlobFSTrash(), - } - jptm.SetStatus(common.ETransferStatus.Started()) - doDeleteHNSResource(jptm) - - a.Nil(err) - a.Equal(jptm.status, common.ETransferStatus.Success()) - -} diff --git a/ste/sender-deleteHNSBlob_test.go b/ste/sender-deleteHNSBlob_test.go new file mode 100644 index 000000000..6ebbe24c1 --- /dev/null +++ b/ste/sender-deleteHNSBlob_test.go @@ -0,0 +1,64 @@ +package ste + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + blobsas "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" + blobservice "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" + "github.com/Azure/azure-storage-azcopy/v10/common" + "github.com/stretchr/testify/assert" +) + +// This function tests the scenario where we return a transfer success even when we receive a 404 response, indicating a resource not found error. +// In this test, we create a container on an HNS enabled account but do not create any blob. This is done to simulate the 404 scenario when attempting to delete a non-existent blob. +// The deletion operation won't find the blob to delete, resulting in a 404 error, and thus returning a transfer success. +func Test404DeleteSuccessLogic(t *testing.T) { + a := assert.New(t) + + // Setup source and destination + accountName, accountKey := getAccountAndKey() + rawURL := fmt.Sprintf("https://%s.dfs.core.windows.net/", accountName) + + credential, err := blob.NewSharedKeyCredential(accountName, accountKey) + a.Nil(err) + client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, &blobservice.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: NewAzcopyHTTPClient(0), + }}) + a.Nil(err) + + cName := generateContainerName() + cc := client.NewContainerClient(cName) + _, err = cc.Create(context.Background(), nil) + a.Nil(err) + defer cc.Delete(context.Background(), nil) + + // Generating the name for a blob without actually creating it. + sourceName := generateBlobName() + + sasURL, err := cc.NewBlobClient(sourceName).GetSASURL( + blobsas.BlobPermissions{Read: true}, + time.Now().Add(1*time.Hour), + nil) + a.Nil(err) + + jptm := &testJobPartTransferManager{ + info: to.Ptr(TransferInfo{ + Source: sasURL, + SrcContainer: cName, + SrcFilePath: sourceName, + }), + fromTo: common.EFromTo.BlobFSTrash(), + } + jptm.SetStatus(common.ETransferStatus.Started()) + doDeleteHNSResource(jptm) + + a.Nil(err) + a.Equal(jptm.status, common.ETransferStatus.Success()) +} From 794186167374a7fdcd53f6ac6f7baeafc3d8c5d5 Mon Sep 17 00:00:00 2001 From: Pranav Malik Date: Wed, 7 Feb 2024 15:46:57 +0530 Subject: [PATCH 5/5] refactoring code --- ste/sender-deleteHNSBlob_test.go | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/ste/sender-deleteHNSBlob_test.go b/ste/sender-deleteHNSBlob_test.go index 6ebbe24c1..76e67b5df 100644 --- a/ste/sender-deleteHNSBlob_test.go +++ b/ste/sender-deleteHNSBlob_test.go @@ -8,16 +8,16 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" - blobsas "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" - blobservice "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/sas" + datalakeservice "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/service" "github.com/Azure/azure-storage-azcopy/v10/common" "github.com/stretchr/testify/assert" ) // This function tests the scenario where we return a transfer success even when we receive a 404 response, indicating a resource not found error. -// In this test, we create a container on an HNS enabled account but do not create any blob. This is done to simulate the 404 scenario when attempting to delete a non-existent blob. -// The deletion operation won't find the blob to delete, resulting in a 404 error, and thus returning a transfer success. +// In this test, we create a container on an HNS enabled account but do not create any file. This is done to simulate the 404 scenario when attempting to delete a non-existent file/directory. +// The deletion operation won't find the file to delete, resulting in a 404 error, and thus returning a transfer success. func Test404DeleteSuccessLogic(t *testing.T) { a := assert.New(t) @@ -25,27 +25,25 @@ func Test404DeleteSuccessLogic(t *testing.T) { accountName, accountKey := getAccountAndKey() rawURL := fmt.Sprintf("https://%s.dfs.core.windows.net/", accountName) - credential, err := blob.NewSharedKeyCredential(accountName, accountKey) + credential, err := azdatalake.NewSharedKeyCredential(accountName, accountKey) a.Nil(err) - client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, &blobservice.ClientOptions{ + + client, err := datalakeservice.NewClientWithSharedKeyCredential(rawURL, credential, &datalakeservice.ClientOptions{ ClientOptions: azcore.ClientOptions{ Transport: NewAzcopyHTTPClient(0), }}) a.Nil(err) cName := generateContainerName() - cc := client.NewContainerClient(cName) + cc := client.NewFileSystemClient(cName) _, err = cc.Create(context.Background(), nil) a.Nil(err) defer cc.Delete(context.Background(), nil) - // Generating the name for a blob without actually creating it. + // Generating the name for a file without actually creating it. sourceName := generateBlobName() - - sasURL, err := cc.NewBlobClient(sourceName).GetSASURL( - blobsas.BlobPermissions{Read: true}, - time.Now().Add(1*time.Hour), - nil) + sasURL, err := client.GetSASURL(sas.AccountResourceTypes{Container: true}, sas.AccountPermissions{Read: true, Add: true, Create: true, Delete: true, Write: true, List: true}, + time.Now().Add(1*time.Hour), nil) a.Nil(err) jptm := &testJobPartTransferManager{