Skip to content

Commit f1121bf

Browse files
gapra-msftgapra-msft
authored
Migrated blobfs smoke tests (#2730)
1 parent 43bb98e commit f1121bf

7 files changed

+164
-522
lines changed

‎common/fe-ste-models.go

+1
Original file line numberDiff line numberDiff line change
@@ -1112,6 +1112,7 @@ const (
11121112
BlockSizeThreshold = 256 * 1024 * 1024
11131113
MinParallelChunkCountThreshold = 4 /* minimum number of chunks in parallel for AzCopy to be performant. */
11141114
MegaByte = 1024 * 1024
1115+
KiloByte = 1024
11151116
)
11161117

11171118
// This struct represent a single transfer entry with source and destination details

‎e2etest/zt_newe2e_blobfs_test.go

+163
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
package e2etest
2+
3+
import (
4+
"github.com/Azure/azure-storage-azcopy/v10/common"
5+
"strconv"
6+
)
7+
8+
func init() {
9+
suiteManager.RegisterSuite(&BlobFSTestSuite{})
10+
}
11+
12+
type BlobFSTestSuite struct{}
13+
14+
func (s *BlobFSTestSuite) Scenario_UploadFile(svm *ScenarioVariationManager) {
15+
fileName := "test.txt"
16+
size := ResolveVariation(svm, []int64{common.KiloByte, 64 * common.MegaByte})
17+
body := NewRandomObjectContentContainer(svm, size)
18+
19+
srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}).
20+
GetObject(svm, fileName, common.EEntityType.File())
21+
srcObj.Create(svm, body, ObjectProperties{})
22+
23+
acct := GetAccount(svm, PrimaryHNSAcct)
24+
dstService := acct.GetService(svm, common.ELocation.BlobFS())
25+
dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{})
26+
27+
RunAzCopy(svm, AzCopyCommand{
28+
Verb: AzCopyVerbCopy,
29+
Targets: []ResourceManager{srcObj, dstContainer.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{})},
30+
Flags: CopyFlags{
31+
CopySyncCommonFlags: CopySyncCommonFlags{
32+
Recursive: pointerTo(true),
33+
},
34+
},
35+
})
36+
37+
ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{
38+
Body: body,
39+
}, true)
40+
}
41+
42+
func (s *BlobFSTestSuite) Scenario_UploadFileMultiflushOAuth(svm *ScenarioVariationManager) {
43+
fileName := "test_multiflush_64MB_file.txt"
44+
body := NewRandomObjectContentContainer(svm, 64*common.MegaByte)
45+
46+
srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}).
47+
GetObject(svm, fileName, common.EEntityType.File())
48+
srcObj.Create(svm, body, ObjectProperties{})
49+
50+
acct := GetAccount(svm, PrimaryHNSAcct)
51+
dstService := acct.GetService(svm, common.ELocation.BlobFS())
52+
dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{})
53+
54+
flushThreshold := ResolveVariation(svm, []uint32{15, 16}) // uneven, even
55+
56+
// Upload the file using AzCopy @ 1MB blocks, 15 block flushes (5 flushes, 4 15 blocks, 1 4 blocks)
57+
RunAzCopy(svm, AzCopyCommand{
58+
Verb: AzCopyVerbCopy,
59+
Targets: []ResourceManager{srcObj, dstContainer.(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.OAuth(), svm, CreateAzCopyTargetOptions{})},
60+
Flags: CopyFlags{
61+
CopySyncCommonFlags: CopySyncCommonFlags{
62+
Recursive: pointerTo(true),
63+
BlockSizeMB: pointerTo(1.0),
64+
},
65+
ADLSFlushThreshold: pointerTo(flushThreshold),
66+
},
67+
})
68+
69+
ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{
70+
Body: body,
71+
}, true)
72+
}
73+
74+
func (s *BlobFSTestSuite) Scenario_Upload100Files(svm *ScenarioVariationManager) {
75+
srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{})
76+
acct := GetAccount(svm, PrimaryHNSAcct)
77+
dstService := acct.GetService(svm, common.ELocation.BlobFS())
78+
dstContainer := CreateResource[ContainerResourceManager](svm, dstService, ResourceDefinitionContainer{})
79+
80+
srcObject := srcContainer.GetObject(svm, "dir_100_files", common.EEntityType.Folder())
81+
82+
srcObjs := make(ObjectResourceMappingFlat)
83+
for i := range 100 {
84+
name := "dir_100_files/test" + strconv.Itoa(i) + ".txt"
85+
obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: NewRandomObjectContentContainer(svm, SizeFromString("1K"))}
86+
CreateResource[ObjectResourceManager](svm, srcContainer, obj)
87+
srcObjs[name] = obj
88+
}
89+
90+
RunAzCopy(svm, AzCopyCommand{
91+
Verb: AzCopyVerbCopy,
92+
Targets: []ResourceManager{srcObject, dstContainer.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{})},
93+
Flags: CopyFlags{
94+
CopySyncCommonFlags: CopySyncCommonFlags{
95+
Recursive: pointerTo(true),
96+
},
97+
},
98+
})
99+
100+
ValidateResource[ContainerResourceManager](svm, dstContainer, ResourceDefinitionContainer{
101+
Objects: srcObjs,
102+
}, true)
103+
}
104+
105+
func (s *BlobFSTestSuite) Scenario_DownloadFile(svm *ScenarioVariationManager) {
106+
fileName := "test.txt"
107+
size := ResolveVariation(svm, []int64{common.KiloByte, 64 * common.MegaByte})
108+
body := NewRandomObjectContentContainer(svm, size)
109+
110+
dstObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}).
111+
GetObject(svm, fileName, common.EEntityType.File())
112+
113+
acct := GetAccount(svm, PrimaryHNSAcct)
114+
srcService := acct.GetService(svm, common.ELocation.BlobFS())
115+
srcContainer := CreateResource[ContainerResourceManager](svm, srcService, ResourceDefinitionContainer{})
116+
srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File())
117+
srcObj.Create(svm, body, ObjectProperties{})
118+
119+
RunAzCopy(svm, AzCopyCommand{
120+
Verb: AzCopyVerbCopy,
121+
Targets: []ResourceManager{srcObj.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{}), dstObj},
122+
Flags: CopyFlags{
123+
CopySyncCommonFlags: CopySyncCommonFlags{
124+
Recursive: pointerTo(true),
125+
},
126+
},
127+
})
128+
129+
ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{
130+
Body: body,
131+
}, true)
132+
}
133+
134+
func (s *BlobFSTestSuite) Scenario_Download100Files(svm *ScenarioVariationManager) {
135+
dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{})
136+
acct := GetAccount(svm, PrimaryHNSAcct)
137+
srcService := acct.GetService(svm, common.ELocation.BlobFS())
138+
srcContainer := CreateResource[ContainerResourceManager](svm, srcService, ResourceDefinitionContainer{})
139+
140+
srcObject := srcContainer.GetObject(svm, "dir_100_files", common.EEntityType.Folder())
141+
142+
srcObjs := make(ObjectResourceMappingFlat)
143+
for i := range 100 {
144+
name := "dir_100_files/test" + strconv.Itoa(i) + ".txt"
145+
obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: NewRandomObjectContentContainer(svm, SizeFromString("1K"))}
146+
CreateResource[ObjectResourceManager](svm, srcContainer, obj)
147+
srcObjs[name] = obj
148+
}
149+
150+
RunAzCopy(svm, AzCopyCommand{
151+
Verb: AzCopyVerbCopy,
152+
Targets: []ResourceManager{srcObject.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{}), dstContainer},
153+
Flags: CopyFlags{
154+
CopySyncCommonFlags: CopySyncCommonFlags{
155+
Recursive: pointerTo(true),
156+
},
157+
},
158+
})
159+
160+
ValidateResource[ContainerResourceManager](svm, dstContainer, ResourceDefinitionContainer{
161+
Objects: srcObjs,
162+
}, true)
163+
}

‎testSuite/scripts/run.py

-8
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,11 @@
55
from test_file_upload import *
66
from test_azcopy_operations import *
77
from test_blobfs_upload_sharedkey import *
8-
from test_blobfs_upload_oauth import *
98
from test_blobfs_download_sharedkey import *
10-
from test_blobfs_download_oauth import *
119
from test_blob_piping import *
1210
from test_blob_sync import *
1311
from test_service_to_service_copy import *
1412
from test_google_cloud_storage_copy import *
15-
from test_blobfs_download_SAS import *
16-
from test_blobfs_upload_SAS import *
1713
from test_file_sync import *
1814
from test_file_copy import *
1915
from test_google_cloud_storage_copy import *
@@ -217,10 +213,6 @@ def main():
217213
Block_Upload_User_Scenarios,
218214
Blob_Download_User_Scenario,
219215
PageBlob_Upload_User_Scenarios,
220-
BlobFs_Upload_OAuth_User_Scenarios,
221-
BlobFs_Download_OAuth_User_Scenarios,
222-
BlobFs_Download_SAS_User_Scenarios,
223-
BlobFs_Upload_SAS_User_Scenarios,
224216
Azcopy_Operation_User_Scenario,
225217
FileShare_Download_User_Scenario,
226218
FileShare_Upload_User_Scenario,

‎testSuite/scripts/test_blobfs_download_SAS.py

-108
This file was deleted.

0 commit comments

Comments
 (0)