-
Notifications
You must be signed in to change notification settings - Fork 231
/
Copy pathazure-pipelines.yml
293 lines (269 loc) · 12.2 KB
/
azure-pipelines.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
trigger:
branches:
include:
- dev
- main
pr:
branches:
include:
- dev
- main
jobs:
- job: Build
timeoutInMinutes: 360
strategy:
matrix:
Ubuntu-20:
imageName: 'ubuntu-20.04'
type: 'linux'
MacOS:
imageName: 'macos-11'
type: 'mac-os'
pool:
vmImage: $(imageName)
steps:
- task: GoTool@0
env:
GO111MODULE: 'on'
inputs:
version: '1.19.2'
- script: |
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.46.2
echo 'Installation complete'
./bin/golangci-lint --version
./bin/golangci-lint run e2etest
displayName: 'Golang Lint Check - Linux'
workingDirectory: $(System.DefaultWorkingDirectory)
condition: eq(variables.type, 'linux')
- script: |
echo 'Running GO Vet'
go vet
displayName: 'Golang Vet - Linux'
workingDirectory: $(System.DefaultWorkingDirectory)
- script: |
GOARCH=amd64 GOOS=linux go build -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_amd64"
displayName: 'Generate Linux AMD64'
condition: eq(variables.type, 'linux')
- script: |
GOARCH=amd64 GOOS=linux go build -tags "se_integration" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64"
displayName: 'Generate Linux AMD64 SE Integration'
condition: eq(variables.type, 'linux')
- script: |
GOARCH=arm64 GOOS=linux go build -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_arm64"
displayName: 'Generate Linux ARM64'
condition: eq(variables.type, 'linux')
- script: |
GOARCH=amd64 GOOS=windows go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_amd64.exe"
displayName: 'Generate Windows AMD64'
condition: eq(variables.type, 'linux')
- script: |
GOARCH=386 GOOS=windows go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_386.exe"
displayName: 'Generate Windows i386'
condition: eq(variables.type, 'linux')
- script: |
GOARCH=arm GOARM=7 GOOS=windows go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe"
displayName: 'Generate Windows ARM'
condition: eq(variables.type, 'linux')
- script: |
cp NOTICE.txt $(Build.ArtifactStagingDirectory)
displayName: 'Copy NOTICE.txt'
condition: eq(variables.type, 'linux')
- script: |
go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64"
displayName: 'Generate MacOS Build with AMD64'
condition: eq(variables.type, 'mac-os')
- script: |
GOARCH=arm64 CGO_ENABLED=1 go build -o "$(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64"
displayName: 'Generate MacOS Build with ARM64'
condition: eq(variables.type, 'mac-os')
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifacts'
condition: succeededOrFailed()
- task: ComponentGovernanceComponentDetection@0
inputs:
scanType: 'Register'
verbosity: 'Verbose'
alertWarningLevel: 'High'
- job: E2E_Test
timeoutInMinutes: 360
# Creating strategies for GOOS: Windows Server 2019 /macOS X Mojave 10.15/Ubuntu 20.04
strategy:
matrix:
Ubuntu-20:
imageName: 'ubuntu-20.04'
type: 'linux'
Windows:
imageName: 'windows-2019'
type: 'windows'
MacOS:
imageName: 'macos-11'
type: 'mac-os'
pool:
vmImage: $(imageName)
steps:
- task: GoTool@0
inputs:
version: '1.19.2'
# Running E2E Tests on Linux - AMD64
- script: |
set -e
GOARCH=amd64 GOOS=linux go build -o azcopy_linux_amd64
export AZCOPY_E2E_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64
go test -timeout 60m -race -short -cover ./e2etest
env:
AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY)
AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME)
AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS)
AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS)
AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID)
AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID)
AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET)
AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME)
AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY)
AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs'
AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG)
AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG)
CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256)
displayName: 'E2E Test Linux - AMD64'
condition: eq(variables.type, 'linux')
# Running E2E Tests on Windows - AMD64
- script: |
go build -o $(System.DefaultWorkingDirectory)/azcopy_windows_amd64.exe
go build -o $(System.DefaultWorkingDirectory)/azcopy_windows_386.exe
echo 'starting E2E tests on windows'
go test -timeout 60m -race -cover -v ./e2etest
env:
AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY)
AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME)
AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS)
AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS)
AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID)
AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID)
AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET)
AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME)
AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY)
AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs'
AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG)
AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG)
CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256)
AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/azcopy_windows_amd64.exe
displayName: 'E2E Test Windows - AMD64'
condition: eq(variables.type, 'windows')
# Running E2E Tests on Mac
- script: |
set -e
go build -o azcopy_darwin_amd64
echo 'starting E2E tests on mac-os'
export AZCOPY_E2E_EXECUTABLE_PATH=$(pwd)/azcopy_darwin_amd64
go test -timeout 60m -race -cover -v ./e2etest
env:
AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY)
AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME)
AZCOPY_E2E_ACCOUNT_KEY_HNS: $(AZCOPY_E2E_ACCOUNT_KEY_HNS)
AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS)
AZCOPY_E2E_TENANT_ID: $(OAUTH_TENANT_ID)
AZCOPY_E2E_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID)
AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET)
AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME)
AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY)
AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs'
AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG)
AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG)
CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256)
displayName: 'E2E Test MacOs AMD64'
condition: eq(variables.type, 'mac-os')
- task: PublishBuildArtifacts@1
displayName: 'Publish logs'
condition: succeededOrFailed()
inputs:
pathToPublish: '$(System.DefaultWorkingDirectory)/logs'
artifactName: logs
- job: Test_On_Ubuntu
variables:
isMutexSet: 'false'
# allow maximum build time, in case we have build congestion
timeoutInMinutes: 360
pool:
vmImage: 'ubuntu-20.04'
steps:
- task: UsePythonVersion@0
name: 'Set_up_Python'
inputs:
versionSpec: '3.7'
- task: GoTool@0
name: 'Set_up_Golang'
inputs:
version: '1.19.2'
- task: DownloadSecureFile@1
name: ciGCSServiceAccountKey
displayName: 'Download GCS Service Account Key'
inputs:
secureFile: 'ci-gcs-dev.json'
- script: |
pip install azure-storage-blob==12.12.0
# set the variable to indicate that the mutex is being acquired
# note: we set it before acquiring the mutex to ensure we release the mutex.
# setting this after can result in an un-broken mutex if someone cancels the pipeline after we acquire the
# mutex but before we set this variable.
# setting this before will always work since it is valid to break an un-acquired mutex.
echo '##vso[task.setvariable variable=isMutexSet]true'
# acquire the mutex before running live tests to avoid conflicts
python ./tool_distributed_mutex.py lock "$(MUTEX_URL)"
name: 'Acquire_the_distributed_mutex'
- script: |
# run unit test and build executable
# the set -e line is needed so that the unit tests failure would cause the job to fail properly
# "-check.v" (must be after package list) outputs timings
set -e
go test -timeout 60m -race -short -cover ./cmd ./common ./common/parallel ./ste ./azbfs ./sddl "-check.v"
GOARCH=amd64 GOOS=linux go build -o azcopy_linux_amd64
name: 'Run_unit_tests'
env:
ACCOUNT_NAME: $(ACCOUNT_NAME)
ACCOUNT_KEY: $(ACCOUNT_KEY)
AZCOPY_E2E_ACCOUNT_KEY: $(AZCOPY_E2E_ACCOUNT_KEY)
AZCOPY_E2E_ACCOUNT_NAME: $(AZCOPY_E2E_ACCOUNT_NAME)
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath)
GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT)
- script: |
go build -o test-validator ./testSuite/
mkdir test-temp
export AZCOPY_EXECUTABLE_PATH=$(pwd)/azcopy_linux_amd64
export TEST_SUITE_EXECUTABLE_LOCATION=$(pwd)/test-validator
export TEST_DIRECTORY_PATH=$(pwd)/test-temp
keyctl session test python ./testSuite/scripts/run.py
name: 'Run_smoke_tests'
env:
ACCOUNT_NAME: $(ACCOUNT_NAME)
ACCOUNT_KEY: $(ACCOUNT_KEY)
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
GOOGLE_CLOUD_PROJECT: $(GOOGLE_CLOUD_PROJECT)
GOOGLE_APPLICATION_CREDENTIALS: $(ciGCSServiceAccountKey.secureFilePath)
ACTIVE_DIRECTORY_APPLICATION_ID: $(ACTIVE_DIRECTORY_APPLICATION_ID)
AZCOPY_SPA_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET)
CONTAINER_OAUTH_URL: $(CONTAINER_OAUTH_URL)
CONTAINER_OAUTH_VALIDATE_SAS_URL: $(CONTAINER_OAUTH_VALIDATE_SAS_URL)
CONTAINER_SAS_URL: $(CONTAINER_SAS_URL)
FILESYSTEM_SAS_URL: $(FILESYSTEM_SAS_URL)
FILESYSTEM_URL: $(FILESYSTEM_URL)
OAUTH_AAD_ENDPOINT: $(OAUTH_AAD_ENDPOINT)
OAUTH_TENANT_ID: $(OAUTH_TENANT_ID)
PREMIUM_CONTAINER_SAS_URL: $(PREMIUM_CONTAINER_SAS_URL)
S2S_DST_BLOB_ACCOUNT_SAS_URL: $(S2S_DST_BLOB_ACCOUNT_SAS_URL)
S2S_SRC_BLOB_ACCOUNT_SAS_URL: $(S2S_SRC_BLOB_ACCOUNT_SAS_URL)
S2S_SRC_FILE_ACCOUNT_SAS_URL: $(S2S_SRC_FILE_ACCOUNT_SAS_URL)
S2S_SRC_S3_SERVICE_URL: $(S2S_SRC_S3_SERVICE_URL)
S2S_SRC_GCP_SERVICE_URL: $(S2S_SRC_GCP_SERVICE_URL)
SHARE_SAS_URL: $(SHARE_SAS_URL)
- script: |
pip install azure-storage-blob==12.12.0
python ./tool_distributed_mutex.py unlock "$(MUTEX_URL)"
name: 'Release_the_distributed_mutex'
# this runs even if the job was canceled (only if the mutex was acquired by this job)
condition: and(always(), eq(variables['isMutexSet'], 'true'))