Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[RCA Repair Item]Retried deletes should ignore 404 errors #2554

Merged
merged 6 commits into from
Feb 7, 2024
Merged

[RCA Repair Item]Retried deletes should ignore 404 errors #2554

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f5245c0
Ignoring the 404 error
pranavmalik-msft Jan 17, 2024
4b7396c
adding 403 support
pranavmalik-msft Jan 18, 2024
fcfb0cb
adding testing support
pranavmalik-msft Jan 26, 2024
d82e671
moving test to new file
pranavmalik-msft Jan 29, 2024
7941861
refactoring code
pranavmalik-msft Feb 7, 2024
1d3c8eb
Merge branch 'main' into pranavmalik/personal/deletehnsfile404fix
pranavmalik-msft Feb 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 55 additions & 5 deletions ste/sender-appendBlob_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ package ste
import (
"context"
"fmt"
"net/http"
"testing"
"time"

"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
Expand All @@ -13,9 +17,6 @@ import (
blobservice "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
"github.com/Azure/azure-storage-azcopy/v10/common"
"github.com/stretchr/testify/assert"
"net/http"
"testing"
"time"
)

type appendErrorInjectionPolicy struct {
Expand Down Expand Up @@ -81,7 +82,7 @@ func Test500FollowedBy412Logic(t *testing.T) {
}),
fromTo: common.EFromTo.BlobBlob(),
}
blobSIP, err := newBlobSourceInfoProvider(jptm)
blobSIP, err := newBlobSourceInfoProvider(&jptm)
a.Nil(err)

injectionPolicy := &appendErrorInjectionPolicy{timedOut: false}
Expand All @@ -92,7 +93,7 @@ func Test500FollowedBy412Logic(t *testing.T) {
},
})
a.Nil(err)
base := appendBlobSenderBase{jptm: jptm, destAppendBlobClient: destClient, sip: blobSIP}
base := appendBlobSenderBase{jptm: &jptm, destAppendBlobClient: destClient, sip: blobSIP}

// Get MD5 range within service calculation
offset := int64(0)
Expand All @@ -108,3 +109,52 @@ func Test500FollowedBy412Logic(t *testing.T) {
a.Nil(err)
a.Empty(errString)
}

// This function tests the scenario where we return a transfer success even when we receive a 404 response, indicating a resource not found error.
// In this test, we create a container on an HNS enabled account but do not create any blob. This is done to simulate the 404 scenario when attempting to delete a non-existent blob.
// The deletion operation won't find the blob to delete, resulting in a 404 error, and thus returning a transfer success.
func Test404DeleteSuccessLogic(t *testing.T) {
a := assert.New(t)

// Setup source and destination
accountName, accountKey := getAccountAndKey()
rawURL := fmt.Sprintf("https://%s.dfs.core.windows.net/", accountName)

credential, err := blob.NewSharedKeyCredential(accountName, accountKey)
a.Nil(err)
client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential, &blobservice.ClientOptions{
ClientOptions: azcore.ClientOptions{
Transport: NewAzcopyHTTPClient(0),
}})
a.Nil(err)

cName := generateContainerName()
cc := client.NewContainerClient(cName)
_, err = cc.Create(context.Background(), nil)
a.Nil(err)
defer cc.Delete(context.Background(), nil)

// Generating the name for a blob without actually creating it.
sourceName := generateBlobName()

sasURL, err := cc.NewBlobClient(sourceName).GetSASURL(
blobsas.BlobPermissions{Read: true},
time.Now().Add(1*time.Hour),
nil)
a.Nil(err)

jptm := &testJobPartTransferManager{
info: to.Ptr(TransferInfo{
Source: sasURL,
SrcContainer: cName,
SrcFilePath: sourceName,
}),
fromTo: common.EFromTo.BlobFSTrash(),
}
jptm.SetStatus(common.ETransferStatus.Started())
doDeleteHNSResource(jptm)

a.Nil(err)
a.Equal(jptm.status, common.ETransferStatus.Success())

}
5 changes: 3 additions & 2 deletions ste/sender_blockBlob_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,14 @@
package ste

import (
"strings"
"testing"

"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
"github.com/Azure/azure-storage-azcopy/v10/common"
"github.com/stretchr/testify/assert"
"strings"
"testing"
)

func TestGetVerifiedChunkParams(t *testing.T) {
Expand Down
33 changes: 17 additions & 16 deletions ste/sourceInfoProvider_md5_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,20 @@ package ste

import (
"bytes"
gcpUtils "cloud.google.com/go/storage"
"context"
"crypto/md5"
"errors"
"fmt"
"hash/crc64"
"io"
"math/rand"
"os"
"runtime"
"strings"
"testing"
"time"

gcpUtils "cloud.google.com/go/storage"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
Expand All @@ -38,14 +47,6 @@ import (
"github.com/Azure/azure-storage-azcopy/v10/common"
"github.com/minio/minio-go"
"github.com/stretchr/testify/assert"
"hash/crc64"
"io"
"math/rand"
"os"
"runtime"
"strings"
"testing"
"time"
)

// This file covers testing of the GetMD5() method in the various sourceInfoProvider implementations
Expand Down Expand Up @@ -174,7 +175,7 @@ func TestBenchmark(t *testing.T) {
info: nil,
fromTo: common.EFromTo.BenchmarkBlob(),
}
benchSIP, err := newBenchmarkSourceInfoProvider(jptm)
benchSIP, err := newBenchmarkSourceInfoProvider(&jptm)
a.Nil(err)

_, err = benchSIP.GetMD5(0, 1)
Expand Down Expand Up @@ -220,7 +221,7 @@ func TestBlockBlob(t *testing.T) {
}),
fromTo: common.EFromTo.BlobBlob(),
}
blobSIP, err := newBlobSourceInfoProvider(jptm)
blobSIP, err := newBlobSourceInfoProvider(&jptm)
a.Nil(err)

// Get MD5 range within service calculation
Expand Down Expand Up @@ -288,7 +289,7 @@ func TestShareFile(t *testing.T) {
}),
fromTo: common.EFromTo.FileBlob(),
}
fileSIP, err := newFileSourceInfoProvider(jptm)
fileSIP, err := newFileSourceInfoProvider(&jptm)
a.Nil(err)

// Get MD5 range within service calculation
Expand Down Expand Up @@ -357,7 +358,7 @@ func TestShareDirectory(t *testing.T) {
}),
fromTo: common.EFromTo.FileBlob(),
}
fileSIP, err := newFileSourceInfoProvider(jptm)
fileSIP, err := newFileSourceInfoProvider(&jptm)
a.Nil(err)

_, err = fileSIP.GetMD5(0, 1)
Expand Down Expand Up @@ -398,7 +399,7 @@ func TestGCP(t *testing.T) {
}),
fromTo: common.EFromTo.GCPBlob(),
}
gcpSIP, err := newGCPSourceInfoProvider(jptm)
gcpSIP, err := newGCPSourceInfoProvider(&jptm)
a.Nil(err)

// Get MD5 range within service calculation
Expand Down Expand Up @@ -448,7 +449,7 @@ func TestLocal(t *testing.T) {
}),
fromTo: common.EFromTo.LocalBlob(),
}
localSIP, err := newLocalSourceInfoProvider(jptm)
localSIP, err := newLocalSourceInfoProvider(&jptm)
a.Nil(err)

// Get MD5 range within service calculation
Expand Down Expand Up @@ -504,7 +505,7 @@ func TestS3(t *testing.T) {
}),
fromTo: common.EFromTo.S3Blob(),
}
s3SIP, err := newS3SourceInfoProvider(jptm)
s3SIP, err := newS3SourceInfoProvider(&jptm)
a.Nil(err)

// Get MD5 range within service calculation
Expand Down
Loading
Loading