Skip to content

Commit 644de7b

Browse files
nakulkar-msftnakulkar-msft
committed
Perf testing for AzCopy (#2006)
* Base commit for perf test * Add tests * Add local tests * Re-enable S2S and use /dev/null for local tests * Add auth mode to Clean destination Containers * Modify bench command to have runtime around 30min * Change image name * Add auth mode to cli command * Change large files count to 50
1 parent 4d47f97 commit 644de7b

File tree

1 file changed

+152
-0
lines changed

1 file changed

+152
-0
lines changed

‎perf-test.yaml

+152
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
trigger: none
2+
pr: none
3+
4+
stages:
5+
- stage: Smallfiles
6+
jobs:
7+
- job: PerformanceTest
8+
timeoutInMinutes: 720
9+
strategy:
10+
matrix:
11+
Ubuntu-22:
12+
imageName: "azcopyPerfTestUbuntu22.04"
13+
Description: "AzCopy Perf Test"
14+
15+
pool:
16+
name: "AzCopyPerfTestUbuntu"
17+
demands:
18+
- ImageOverride -equals $(imageName)
19+
20+
variables:
21+
- group: AzCopyPerfTestTargets
22+
- name: localPath
23+
value: "/mnt/storage"
24+
25+
steps:
26+
- script: |
27+
echo $(Description)
28+
hostnamectl
29+
displayName: 'Print Agent Info'
30+
31+
- task: GoTool@0
32+
inputs:
33+
version: '1.19.3'
34+
35+
- script: |
36+
go build -o $GOROOT/bin/azcopy
37+
azcopy --version
38+
displayName: 'Build Azcopy'
39+
40+
- script: |
41+
time azcopy copy $(Blob2BlobLargeFilesSrc) $(Blob2BlobLargeFilesDst) --recursive --block-size-mb=128 --log-level=ERROR --cap-mbps=40000
42+
displayName: 'Blob2Blob - Large Files'
43+
condition: always()
44+
env:
45+
AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
46+
AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
47+
AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs
48+
AZCOPY_CONCURRENCY_VALUE: "256"
49+
AZCOPY_SHOW_PERF_STATES: "1"
50+
51+
- script: |
52+
time azcopy copy $(Blob2BlobSmallAndMedFilesSrc) $(Blob2BlobSmallAndMedFilesDst) --recursive --block-size-mb=128 --log-level=ERROR
53+
displayName: 'Blob2Blob - Small to Medium sized files'
54+
condition: always()
55+
env:
56+
AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
57+
AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
58+
AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs
59+
AZCOPY_CONCURRENCY_VALUE: "256"
60+
AZCOPY_SHOW_PERF_STATES: "1"
61+
62+
- script: |
63+
time azcopy copy $(Blob2BlobSmallFilesSrc) $(Blob2BlobSmallFilesDst) --recursive --check-length=false --log-level=ERROR
64+
displayName: 'Blob2Blob - Small Files'
65+
condition: always()
66+
env:
67+
AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
68+
AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
69+
AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs
70+
AZCOPY_CONCURRENCY_VALUE: "256"
71+
AZCOPY_SHOW_PERF_STATES: "1"
72+
73+
- script: |
74+
sudo mkdir -m 777 $(localPath)/largeFiles/
75+
time azcopy copy $(Blob2BlobLargeFilesSrc) /dev/null --recursive --log-level=ERROR
76+
displayName: 'Download - Large files'
77+
condition: always()
78+
env:
79+
AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
80+
AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
81+
AZCOPY_SHOW_PERF_STATES: "1"
82+
AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs
83+
84+
- script: |
85+
time azcopy bench $(Blob2BlobLargeFilesDst) --log-level=ERROR --size-per-file=50G --file-count=50 --put-md5=false --delete-test-data=false
86+
sudo rm -rf $(localPath)/*
87+
displayName: 'Upload - Large files'
88+
condition: always()
89+
env:
90+
AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
91+
AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
92+
AZCOPY_SHOW_PERF_STATES: "1"
93+
AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs
94+
95+
- script: |
96+
sudo mkdir -m 777 $(localPath)/smallToMediumFiles/
97+
time azcopy copy $(Blob2BlobSmallAndMedFilesSrc) $(localPath)/smallToMediumFiles --recursive --log-level=ERROR
98+
displayName: 'Download - Small to Medium sized files'
99+
condition: always()
100+
env:
101+
AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
102+
AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
103+
AZCOPY_SHOW_PERF_STATES: "1"
104+
AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs
105+
106+
- script: |
107+
time azcopy copy $(localPath)/smallToMediumFiles/ $(Blob2BlobSmallAndMedFilesDst) --recursive --log-level=ERROR
108+
sudo rm -rf $(localPath)/*
109+
displayName: 'Upload - Small to Medium sized files'
110+
condition: always()
111+
env:
112+
AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
113+
AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
114+
AZCOPY_SHOW_PERF_STATES: "1"
115+
AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs
116+
117+
- script: |
118+
sudo mkdir -m 777 $(localPath)/smallFiles/
119+
time azcopy copy $(Blob2BlobSmallFilesSrc) /dev/null --recursive --check-length=false --log-level=ERROR
120+
displayName: 'Download - Small Files'
121+
condition: always()
122+
env:
123+
AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
124+
AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
125+
AZCOPY_SHOW_PERF_STATES: "1"
126+
AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs
127+
128+
- script: |
129+
time azcopy bench $(Blob2BlobSmallFilesDst) --size-per-file=5k --file-count=8000000 --check-length=false --log-level=ERROR --delete-test-data=false
130+
sudo rm -rf $(localPath)/*
131+
displayName: 'Upload - Small Files'
132+
condition: always()
133+
env:
134+
AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
135+
AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
136+
AZCOPY_SHOW_PERF_STATES: "1"
137+
AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs
138+
139+
- task: PublishBuildArtifacts@1
140+
condition: always()
141+
inputs:
142+
pathToPublish: $(Build.ArtifactStagingDirectory)
143+
artifactName: Logs
144+
145+
- script: |
146+
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
147+
az login --identity --username $(AZCOPY_MSI_CLIENT_ID)
148+
for container in `az storage container list --account-name $(DestinationAccount) --query "[*].[name]" --output tsv --auth-mode login`; do
149+
az storage container delete --account-name $(DestinationAccount) --name $container --auth-mode login
150+
done
151+
displayName: 'Clean destination storage Account'
152+
condition: always()

0 commit comments

Comments
 (0)