Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrated blobfs smoke tests #2730

Merged
merged 9 commits into from
Jul 17, 2024
Merged

Migrated blobfs smoke tests #2730

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
2b5a980
Migrated smoke tests for auto detect blob type
gapra-msft Jun 20, 2024
0fb89b9
Migrated blobfs smoke tests
gapra-msft Jun 20, 2024
f08026a
Merge branch 'main' into gapra/smokeTestAutoDetectBlobType
gapra-msft Jun 20, 2024
04c1b60
Merge branch 'gapra/smokeTestAutoDetectBlobType' into gapra/smokeTest…
gapra-msft Jun 20, 2024
7924036
Merge branch 'main' into gapra/smokeTestAutoDetectBlobType
gapra-msft Jul 15, 2024
08e6f7f
combine tests
gapra-msft Jul 16, 2024
0a6a619
Merge branch 'gapra/smokeTestAutoDetectBlobType' into gapra/smokeTest…
gapra-msft Jul 16, 2024
aeb6640
Addressed comments
gapra-msft Jul 16, 2024
4b26495
Merge branch 'main' into gapra/smokeTestBlobFs
gapra-msft Jul 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions common/fe-ste-models.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1112,6 +1112,7 @@ const (
BlockSizeThreshold = 256 * 1024 * 1024
MinParallelChunkCountThreshold = 4 /* minimum number of chunks in parallel for AzCopy to be performant. */
MegaByte = 1024 * 1024
KiloByte = 1024
)

// This struct represent a single transfer entry with source and destination details
Expand Down
109 changes: 109 additions & 0 deletions e2etest/zt_newe2e_autodetect_blob_type_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
package e2etest

import (
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-storage-azcopy/v10/common"
)

func init() {
suiteManager.RegisterSuite(&AutoDetectBlobTypeTestSuite{})
}

type AutoDetectBlobTypeTestSuite struct{}

func (s *AutoDetectBlobTypeTestSuite) Scenario_AutoInferBlobTypeVHD(svm *ScenarioVariationManager) {
fileName := "myVHD.vHd" // awkward capitalization to see if AzCopy catches it.
body := NewRandomObjectContentContainer(svm, 4*common.MegaByte)

srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}).
GetObject(svm, fileName, common.EEntityType.File())
srcObj.Create(svm, body, ObjectProperties{})

dstObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}).
GetObject(svm, fileName, common.EEntityType.File())

// copy vhd file without specifying page blob. Page blob is inferred for VHD, VHDX, and VMDK
RunAzCopy(svm, AzCopyCommand{
Verb: AzCopyVerbCopy,
Targets: []ResourceManager{srcObj, dstObj},
Flags: CopyFlags{
CopySyncCommonFlags: CopySyncCommonFlags{
Recursive: pointerTo(true),
BlockSizeMB: pointerTo(4.0),
},
},
})

ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{
Body: body,
ObjectProperties: ObjectProperties{
BlobProperties: BlobProperties{
Type: pointerTo(blob.BlobTypePageBlob),
},
},
}, true)
}

func (s *AutoDetectBlobTypeTestSuite) Scenario_InferBlobTypeFilePageBlob(svm *ScenarioVariationManager) {
fileName := "testS2SVHD.vhd"
body := NewRandomObjectContentContainer(svm, 4*common.MegaByte)

srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{})
srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File())
srcObj.Create(svm, body, ObjectProperties{})

dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{})

RunAzCopy(svm, AzCopyCommand{
Verb: AzCopyVerbCopy,
Targets: []ResourceManager{srcContainer, dstContainer},
Flags: CopyFlags{
CopySyncCommonFlags: CopySyncCommonFlags{
Recursive: pointerTo(true),
},
},
})

ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{
Body: body,
ObjectProperties: ObjectProperties{
BlobProperties: BlobProperties{
Type: pointerTo(blob.BlobTypePageBlob),
},
},
}, true)
}

func (s *AutoDetectBlobTypeTestSuite) Scenario_DetectBlobTypeBlobBlob(svm *ScenarioVariationManager) {
fileName := "testS2SVHD.vhd"
body := NewRandomObjectContentContainer(svm, 4*common.MegaByte)

// Upload to Azure Blob Storage as Block Blob and detect as Block Blob.
// AzCopy detects the source blob type.
// This means that in all scenarios EXCEPT Blob -> Blob, .vhd corresponds to page blob.
// However, in Blob -> Blob, we preserve the blob type instead of detecting it.
srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{})
srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File())
srcObj.Create(svm, body, ObjectProperties{})

dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{})

RunAzCopy(svm, AzCopyCommand{
Verb: AzCopyVerbCopy,
Targets: []ResourceManager{srcContainer, dstContainer},
Flags: CopyFlags{
CopySyncCommonFlags: CopySyncCommonFlags{
Recursive: pointerTo(true),
},
},
})

ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{
Body: body,
ObjectProperties: ObjectProperties{
BlobProperties: BlobProperties{
Type: pointerTo(blob.BlobTypeBlockBlob),
},
},
}, true)
}
244 changes: 244 additions & 0 deletions e2etest/zt_newe2e_blobfs_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,244 @@
package e2etest

import (
"github.com/Azure/azure-storage-azcopy/v10/common"
"strconv"
)

func init() {
suiteManager.RegisterSuite(&BlobFSTestSuite{})
}

type BlobFSTestSuite struct{}

func (s *BlobFSTestSuite) Scenario_UploadFile(svm *ScenarioVariationManager) {
fileName := "test_1k.txt"
body := NewRandomObjectContentContainer(svm, common.KiloByte)

srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}).
GetObject(svm, fileName, common.EEntityType.File())
srcObj.Create(svm, body, ObjectProperties{})

acct := GetAccount(svm, PrimaryHNSAcct)
dstService := acct.GetService(svm, common.ELocation.BlobFS())
dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{})

RunAzCopy(svm, AzCopyCommand{
Verb: AzCopyVerbCopy,
Targets: []ResourceManager{srcObj, dstContainer.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{})},
Flags: CopyFlags{
CopySyncCommonFlags: CopySyncCommonFlags{
Recursive: pointerTo(true),
},
},
})

ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{
Body: body,
}, true)
}

func (s *BlobFSTestSuite) Scenario_UploadFile64MB(svm *ScenarioVariationManager) {
fileName := "test_64mb.txt"
body := NewRandomObjectContentContainer(svm, 64*common.MegaByte)

srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}).
GetObject(svm, fileName, common.EEntityType.File())
srcObj.Create(svm, body, ObjectProperties{})

acct := GetAccount(svm, PrimaryHNSAcct)
dstService := acct.GetService(svm, common.ELocation.BlobFS())
dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{})

RunAzCopy(svm, AzCopyCommand{
Verb: AzCopyVerbCopy,
Targets: []ResourceManager{srcObj, dstContainer.(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{})},
Flags: CopyFlags{
CopySyncCommonFlags: CopySyncCommonFlags{
Recursive: pointerTo(true),
},
},
})

ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{
Body: body,
}, true)
}

func (s *BlobFSTestSuite) Scenario_UploadFileUnevenMultiflushOAuth(svm *ScenarioVariationManager) {
fileName := "test_uneven_multiflush_64MB_file.txt"
body := NewRandomObjectContentContainer(svm, 64*common.MegaByte)

srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}).
GetObject(svm, fileName, common.EEntityType.File())
srcObj.Create(svm, body, ObjectProperties{})

acct := GetAccount(svm, PrimaryHNSAcct)
dstService := acct.GetService(svm, common.ELocation.BlobFS())
dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{})

// Upload the file using AzCopy @ 1MB blocks, 15 block flushes (5 flushes, 4 15 blocks, 1 4 blocks)
RunAzCopy(svm, AzCopyCommand{
Verb: AzCopyVerbCopy,
Targets: []ResourceManager{srcObj, dstContainer.(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.OAuth(), svm, CreateAzCopyTargetOptions{})},
Flags: CopyFlags{
CopySyncCommonFlags: CopySyncCommonFlags{
Recursive: pointerTo(true),
BlockSizeMB: pointerTo(1.0),
},
ADLSFlushThreshold: pointerTo(uint32(15)),
},
})

ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{
Body: body,
}, true)
}

func (s *BlobFSTestSuite) Scenario_UploadFileEvenMultiflushOAuth(svm *ScenarioVariationManager) {
fileName := "test_even_multiflush_64MB_file.txt"
body := NewRandomObjectContentContainer(svm, 64*common.MegaByte)

srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}).
GetObject(svm, fileName, common.EEntityType.File())
srcObj.Create(svm, body, ObjectProperties{})

acct := GetAccount(svm, PrimaryHNSAcct)
dstService := acct.GetService(svm, common.ELocation.BlobFS())
dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{})

// Upload the file using AzCopy @ 1MB blocks, 16 block flushes (4 16 block flushes)
RunAzCopy(svm, AzCopyCommand{
Verb: AzCopyVerbCopy,
Targets: []ResourceManager{srcObj, dstContainer.(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.OAuth(), svm, CreateAzCopyTargetOptions{})},
Flags: CopyFlags{
CopySyncCommonFlags: CopySyncCommonFlags{
Recursive: pointerTo(true),
BlockSizeMB: pointerTo(1.0),
},
ADLSFlushThreshold: pointerTo(uint32(16)),
},
})

ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{
Body: body,
}, true)
}

func (s *BlobFSTestSuite) Scenario_Upload100Files(svm *ScenarioVariationManager) {
srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{})
acct := GetAccount(svm, PrimaryHNSAcct)
dstService := acct.GetService(svm, common.ELocation.BlobFS())
dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{})

srcObject := srcContainer.GetObject(svm, "dir_100_files", common.EEntityType.Folder())

srcObjs := make(ObjectResourceMappingFlat)
for i := range 100 {
name := "dir_100_files/test" + strconv.Itoa(i) + ".txt"
obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: NewRandomObjectContentContainer(svm, SizeFromString("1K"))}
CreateResource[ObjectResourceManager](svm, srcContainer, obj)
srcObjs[name] = obj
}

RunAzCopy(svm, AzCopyCommand{
Verb: AzCopyVerbCopy,
Targets: []ResourceManager{srcObject, dstContainer.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{})},
Flags: CopyFlags{
CopySyncCommonFlags: CopySyncCommonFlags{
Recursive: pointerTo(true),
},
},
})

ValidateResource[ContainerResourceManager](svm, dstContainer, ResourceDefinitionContainer{
Objects: srcObjs,
}, true)
}

func (s *BlobFSTestSuite) Scenario_DownloadFile(svm *ScenarioVariationManager) {
fileName := "test_1k.txt"
body := NewRandomObjectContentContainer(svm, common.KiloByte)

dstObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}).
GetObject(svm, fileName, common.EEntityType.File())

acct := GetAccount(svm, PrimaryHNSAcct)
srcService := acct.GetService(svm, common.ELocation.BlobFS())
srcContainer := CreateResource[ContainerResourceManager](svm, srcService, ResourceDefinitionContainer{})
srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File())
srcObj.Create(svm, body, ObjectProperties{})

RunAzCopy(svm, AzCopyCommand{
Verb: AzCopyVerbCopy,
Targets: []ResourceManager{srcObj.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{}), dstObj},
Flags: CopyFlags{
CopySyncCommonFlags: CopySyncCommonFlags{
Recursive: pointerTo(true),
},
},
})

ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{
Body: body,
}, true)
}

func (s *BlobFSTestSuite) Scenario_DownloadFile64MB(svm *ScenarioVariationManager) {
fileName := "test_64mb.txt"
body := NewRandomObjectContentContainer(svm, common.KiloByte)

dstObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}).
GetObject(svm, fileName, common.EEntityType.File())

acct := GetAccount(svm, PrimaryHNSAcct)
srcService := acct.GetService(svm, common.ELocation.BlobFS())
srcContainer := CreateResource[ContainerResourceManager](svm, srcService, ResourceDefinitionContainer{})
srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File())
srcObj.Create(svm, body, ObjectProperties{})

RunAzCopy(svm, AzCopyCommand{
Verb: AzCopyVerbCopy,
Targets: []ResourceManager{srcObj.(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{}), dstObj},
Flags: CopyFlags{
CopySyncCommonFlags: CopySyncCommonFlags{
Recursive: pointerTo(true),
},
},
})

ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{
Body: body,
}, true)
}

func (s *BlobFSTestSuite) Scenario_Download100Files(svm *ScenarioVariationManager) {
dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{})
acct := GetAccount(svm, PrimaryHNSAcct)
srcService := acct.GetService(svm, common.ELocation.BlobFS())
srcContainer := CreateResource[ContainerResourceManager](svm, srcService, ResourceDefinitionContainer{})

srcObject := srcContainer.GetObject(svm, "dir_100_files", common.EEntityType.Folder())

srcObjs := make(ObjectResourceMappingFlat)
for i := range 100 {
name := "dir_100_files/test" + strconv.Itoa(i) + ".txt"
obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: NewRandomObjectContentContainer(svm, SizeFromString("1K"))}
CreateResource[ObjectResourceManager](svm, srcContainer, obj)
srcObjs[name] = obj
}

RunAzCopy(svm, AzCopyCommand{
Verb: AzCopyVerbCopy,
Targets: []ResourceManager{srcObject.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{}), dstContainer},
Flags: CopyFlags{
CopySyncCommonFlags: CopySyncCommonFlags{
Recursive: pointerTo(true),
},
},
})

ValidateResource[ContainerResourceManager](svm, dstContainer, ResourceDefinitionContainer{
Objects: srcObjs,
}, true)
}
10 changes: 0 additions & 10 deletions testSuite/scripts/run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,11 @@
from test_file_upload import *
from test_azcopy_operations import *
from test_blobfs_upload_sharedkey import *
from test_blobfs_upload_oauth import *
from test_blobfs_download_sharedkey import *
from test_blobfs_download_oauth import *
from test_blob_piping import *
from test_blob_sync import *
from test_service_to_service_copy import *
from test_google_cloud_storage_copy import *
from test_blobfs_download_SAS import *
from test_blobfs_upload_SAS import *
from test_autodetect_blob_type import *
from test_file_sync import *
from test_file_copy import *
from test_google_cloud_storage_copy import *
Expand Down Expand Up @@ -218,17 +213,12 @@ def main():
Block_Upload_User_Scenarios,
Blob_Download_User_Scenario,
PageBlob_Upload_User_Scenarios,
BlobFs_Upload_OAuth_User_Scenarios,
BlobFs_Download_OAuth_User_Scenarios,
BlobFs_Download_SAS_User_Scenarios,
BlobFs_Upload_SAS_User_Scenarios,
Azcopy_Operation_User_Scenario,
FileShare_Download_User_Scenario,
FileShare_Upload_User_Scenario,
BlobFs_Upload_ShareKey_User_Scenarios,
BlobFs_Download_SharedKey_User_Scenarios,
Service_2_Service_Copy_User_Scenario,
Autodetect_Blob_Type_Scenario,
Google_Cloud_Storage_Copy_User_Scenario]
suites_list = []

Expand Down
Loading