Skip to content

Commit e376658

Browse files
authored
Improve debug-ability of e2e tests by uploading logs of failed jobs (#1898)
* Upload testing logs to storage account on failed test * Handle as pipeline artifact instead * mkdirall * copy plan files too * Fix failing tests * Change overwrite to affect any "locked in"/completed state
1 parent 052b801 commit e376658

File tree

6 files changed

+82
-21
lines changed

6 files changed

+82
-21
lines changed

azure-pipelines.yml

+10
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ jobs:
139139
AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET)
140140
AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME)
141141
AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY)
142+
AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs'
142143
AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG)
143144
AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG)
144145
CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
@@ -162,6 +163,7 @@ jobs:
162163
AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET)
163164
AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME)
164165
AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY)
166+
AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs'
165167
AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG)
166168
AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG)
167169
CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
@@ -187,13 +189,21 @@ jobs:
187189
AZCOPY_E2E_CLIENT_SECRET: $(AZCOPY_SPA_CLIENT_SECRET)
188190
AZCOPY_E2E_CLASSIC_ACCOUNT_NAME: $(AZCOPY_E2E_CLASSIC_ACCOUNT_NAME)
189191
AZCOPY_E2E_CLASSIC_ACCOUNT_KEY: $(AZCOPY_E2E_CLASSIC_ACCOUNT_KEY)
192+
AZCOPY_E2E_LOG_OUTPUT: '$(System.DefaultWorkingDirectory)/logs'
190193
AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG)
191194
AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG)
192195
CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
193196
CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256)
194197
displayName: 'E2E Test MacOs AMD64'
195198
condition: eq(variables.type, 'mac-os')
196199
200+
- task: PublishBuildArtifacts@1
201+
displayName: 'Publish logs'
202+
condition: succeededOrFailed()
203+
inputs:
204+
pathToPublish: '$(System.DefaultWorkingDirectory)/logs'
205+
artifactName: logs
206+
197207
- job: Test_On_Ubuntu
198208
variables:
199209
isMutexSet: 'false'

common/fe-ste-models.go

+5
Original file line numberDiff line numberDiff line change
@@ -670,6 +670,11 @@ var ETransferStatus = TransferStatus(0)
670670

671671
type TransferStatus int32 // Must be 32-bit for atomic operations; negative #s represent a specific failure code
672672

673+
func (t TransferStatus) StatusLocked() bool { // Is an overwrite necessary to change tx status?
674+
// Any kind of failure, or success is considered "locked in".
675+
return t <= ETransferStatus.Failed() || t == ETransferStatus.Success()
676+
}
677+
673678
// Transfer is ready to transfer and not started transferring yet
674679
func (TransferStatus) NotStarted() TransferStatus { return TransferStatus(0) }
675680

e2etest/declarativeScenario.go

+45-11
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,31 @@ type scenarioState struct {
7070
func (s *scenario) Run() {
7171
defer s.cleanup()
7272

73+
// setup runner
74+
azcopyDir, err := os.MkdirTemp("", "")
75+
if err != nil {
76+
s.a.Error(err.Error())
77+
return
78+
}
79+
azcopyRan := false
80+
defer func() {
81+
if os.Getenv("AZCOPY_E2E_LOG_OUTPUT") == "" {
82+
s.a.Assert(os.RemoveAll(azcopyDir), equals(), nil)
83+
return // no need, just delete logdir
84+
}
85+
86+
err := os.MkdirAll(os.Getenv("AZCOPY_E2E_LOG_OUTPUT"), os.ModePerm|os.ModeDir)
87+
if err != nil {
88+
s.a.Assert(err, equals(), nil)
89+
return
90+
}
91+
if azcopyRan && s.a.Failed() {
92+
s.uploadLogs(azcopyDir)
93+
s.a.(*testingAsserter).t.Log("uploaded logs for job " + s.state.result.jobID.String() + " as an artifact")
94+
}
95+
}()
96+
97+
// setup scenario
7398
// First, validate the accounts make sense for the source/dests
7499
if s.srcAccountType.IsBlobOnly() {
75100
s.a.Assert(s.fromTo.From(), equals(), common.ELocation.Blob())
@@ -97,29 +122,30 @@ func (s *scenario) Run() {
97122
}
98123

99124
// execute
100-
s.runAzCopy()
125+
azcopyRan = true
126+
s.runAzCopy(azcopyDir)
101127
if s.a.Failed() {
102128
return // execution failed. No point in running validation
103129
}
104130

105131
// resume if needed
106132
if s.needResume {
107-
tx, err := s.state.result.GetTransferList(common.ETransferStatus.Cancelled())
133+
tx, err := s.state.result.GetTransferList(common.ETransferStatus.Cancelled(), azcopyDir)
108134
s.a.AssertNoErr(err, "Failed to get transfer list for Cancelled")
109135
s.a.Assert(len(tx), equals(), len(s.p.debugSkipFiles), "Job cancel didn't completely work")
110136

111137
if !s.runHook(s.hs.beforeResumeHook) {
112138
return
113139
}
114140

115-
s.resumeAzCopy()
141+
s.resumeAzCopy(azcopyDir)
116142
}
117143
if s.a.Failed() {
118144
return // resume failed. No point in running validation
119145
}
120146

121147
// check
122-
s.validateTransferStates()
148+
s.validateTransferStates(azcopyDir)
123149
if s.a.Failed() {
124150
return // no point in doing more validation
125151
}
@@ -138,6 +164,13 @@ func (s *scenario) Run() {
138164
s.runHook(s.hs.afterValidation)
139165
}
140166

167+
func (s *scenario) uploadLogs(logDir string) {
168+
if s.state.result == nil || os.Getenv("AZCOPY_E2E_LOG_OUTPUT") == "" {
169+
return // nothing to upload
170+
}
171+
s.a.Assert(os.Rename(logDir, filepath.Join(os.Getenv("AZCOPY_E2E_LOG_OUTPUT"), s.state.result.jobID.String())), equals(), nil)
172+
}
173+
141174
func (s *scenario) runHook(h hookFunc) bool {
142175
if h == nil {
143176
return true // nothing to do. So "successful"
@@ -195,7 +228,7 @@ func (s *scenario) assignSourceAndDest() {
195228
s.state.dest = createTestResource(s.fromTo.To(), false)
196229
}
197230

198-
func (s *scenario) runAzCopy() {
231+
func (s *scenario) runAzCopy(logDirectory string) {
199232
s.chToStdin = make(chan string) // unubuffered seems the most predictable for our usages
200233
defer close(s.chToStdin)
201234

@@ -223,9 +256,9 @@ func (s *scenario) runAzCopy() {
223256
result, wasClean, err := r.ExecuteAzCopyCommand(
224257
s.operation,
225258
s.state.source.getParam(s.stripTopDir, needsSAS(s.credTypes[0]), tf.objectTarget),
226-
s.state.dest.getParam(false, needsSAS(s.credTypes[1]), common.IffString(tf.destTarget != "", tf.destTarget, tf.objectTarget)),
227-
s.credTypes[0].IsAzureOAuth() || s.credTypes[1].IsAzureOAuth(), // needsOAuth
228-
afterStart, s.chToStdin)
259+
s.state.dest.getParam(false, needsSAS(s.credTypes[1]), common.IffString(tf.destTarget != "", tf.destTarget, tf.objectTarget)),
260+
s.credTypes[0] == common.ECredentialType.OAuthToken() || s.credTypes[1] == common.ECredentialType.OAuthToken(), // needsOAuth
261+
afterStart, s.chToStdin, logDirectory)
229262

230263
if !wasClean {
231264
s.a.AssertNoErr(err, "running AzCopy")
@@ -243,7 +276,7 @@ func (s *scenario) runAzCopy() {
243276
s.state.result = &result
244277
}
245278

246-
func (s *scenario) resumeAzCopy() {
279+
func (s *scenario) resumeAzCopy(logDir string) {
247280
s.chToStdin = make(chan string) // unubuffered seems the most predictable for our usages
248281
defer close(s.chToStdin)
249282

@@ -274,6 +307,7 @@ func (s *scenario) resumeAzCopy() {
274307
false,
275308
afterStart,
276309
s.chToStdin,
310+
logDir,
277311
)
278312

279313
if !wasClean {
@@ -295,7 +329,7 @@ func (s *scenario) validateRemove() {
295329
}
296330
}
297331
}
298-
func (s *scenario) validateTransferStates() {
332+
func (s *scenario) validateTransferStates(azcopyDir string) {
299333
if s.operation == eOperation.Remove() {
300334
s.validateRemove()
301335
return
@@ -318,7 +352,7 @@ func (s *scenario) validateTransferStates() {
318352
// Is that OK? (Not sure what to do if it's not, because azcopy jobs show, apparently doesn't offer us a way to get the skipped list)
319353
} {
320354
expectedTransfers := s.fs.getForStatus(statusToTest, expectFolders, expectRootFolder)
321-
actualTransfers, err := s.state.result.GetTransferList(statusToTest)
355+
actualTransfers, err := s.state.result.GetTransferList(statusToTest, azcopyDir)
322356
s.a.AssertNoErr(err)
323357

324358
Validator{}.ValidateCopyTransfersAreScheduled(s.a, isSrcEncoded, isDstEncoded, srcRoot, dstRoot, expectedTransfers, actualTransfers, statusToTest, s.FromTo(), s.srcAccountType, s.destAccountType)

e2etest/runner.go

+17-5
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"fmt"
2727
"os"
2828
"os/exec"
29+
"path/filepath"
2930
"strconv"
3031
"strings"
3132

@@ -184,7 +185,7 @@ func (t *TestRunner) execDebuggableWithOutput(name string, args []string, env []
184185
return stdout.Bytes(), runErr
185186
}
186187

187-
func (t *TestRunner) ExecuteAzCopyCommand(operation Operation, src, dst string, needsOAuth bool, afterStart func() string, chToStdin <-chan string) (CopyOrSyncCommandResult, bool, error) {
188+
func (t *TestRunner) ExecuteAzCopyCommand(operation Operation, src, dst string, needsOAuth bool, afterStart func() string, chToStdin <-chan string, logDir string) (CopyOrSyncCommandResult, bool, error) {
188189
capLen := func(b []byte) []byte {
189190
if len(b) < 1024 {
190191
return b
@@ -234,6 +235,11 @@ func (t *TestRunner) ExecuteAzCopyCommand(operation Operation, src, dst string,
234235
}
235236
}
236237

238+
if logDir != "" {
239+
env = append(env, "AZCOPY_LOG_LOCATION="+logDir)
240+
env = append(env, "AZCOPY_JOB_PLAN_LOCATION="+filepath.Join(logDir, "plans"))
241+
}
242+
237243
out, err := t.execDebuggableWithOutput(GlobalInputManager{}.GetExecutablePath(), args, env, afterStart, chToStdin)
238244

239245
wasClean := true
@@ -268,9 +274,15 @@ func (t *TestRunner) SetTransferStatusFlag(value string) {
268274
t.flags["with-status"] = value
269275
}
270276

271-
func (t *TestRunner) ExecuteJobsShowCommand(jobID common.JobID) (JobsShowCommandResult, error) {
277+
func (t *TestRunner) ExecuteJobsShowCommand(jobID common.JobID, azcopyDir string) (JobsShowCommandResult, error) {
272278
args := append([]string{"jobs", "show", jobID.String()}, t.computeArgs()...)
273-
out, err := exec.Command(GlobalInputManager{}.GetExecutablePath(), args...).Output()
279+
cmd := exec.Command(GlobalInputManager{}.GetExecutablePath(), args...)
280+
281+
if azcopyDir != "" {
282+
cmd.Env = append(cmd.Env, "AZCOPY_JOB_PLAN_LOCATION="+filepath.Join(azcopyDir, "plans"))
283+
}
284+
285+
out, err := cmd.Output()
274286
if err != nil {
275287
return JobsShowCommandResult{}, err
276288
}
@@ -307,12 +319,12 @@ func newCopyOrSyncCommandResult(rawOutput string) (CopyOrSyncCommandResult, bool
307319
return CopyOrSyncCommandResult{jobID: jobSummary.JobID, finalStatus: jobSummary}, true
308320
}
309321

310-
func (c *CopyOrSyncCommandResult) GetTransferList(status common.TransferStatus) ([]common.TransferDetail, error) {
322+
func (c *CopyOrSyncCommandResult) GetTransferList(status common.TransferStatus, azcopyDir string) ([]common.TransferDetail, error) {
311323
runner := newTestRunner()
312324
runner.SetTransferStatusFlag(status.String())
313325

314326
// invoke AzCopy to get the status from the plan files
315-
result, err := runner.ExecuteJobsShowCommand(c.jobID)
327+
result, err := runner.ExecuteJobsShowCommand(c.jobID, azcopyDir)
316328
if err != nil {
317329
return make([]common.TransferDetail, 0), err
318330
}

main.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,13 @@ func main() {
6060
// the user can optionally put the plan files somewhere else
6161
if azcopyJobPlanFolder == "" {
6262
// make the app path folder ".azcopy" first so we can make a plans folder in it
63-
if err := os.Mkdir(azcopyAppPathFolder, os.ModeDir); err != nil && !os.IsExist(err) {
63+
if err := os.MkdirAll(azcopyAppPathFolder, os.ModeDir); err != nil && !os.IsExist(err) {
6464
common.PanicIfErr(err)
6565
}
6666
azcopyJobPlanFolder = path.Join(azcopyAppPathFolder, "plans")
6767
}
68-
if err := os.Mkdir(azcopyJobPlanFolder, os.ModeDir|os.ModePerm); err != nil && !os.IsExist(err) {
68+
69+
if err := os.MkdirAll(azcopyJobPlanFolder, os.ModeDir|os.ModePerm); err != nil && !os.IsExist(err) {
6970
log.Fatalf("Problem making .azcopy directory. Try setting AZCOPY_PLAN_FILE_LOCATION env variable. %v", err)
7071
}
7172

ste/JobPartPlan.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -409,9 +409,8 @@ func (jppt *JobPartPlanTransfer) SetTransferStatus(status common.TransferStatus,
409409
if !overWrite {
410410
common.AtomicMorphInt32((*int32)(&jppt.atomicTransferStatus),
411411
func(startVal int32) (val int32, morphResult interface{}) {
412-
// start value < 0 means that transfer status is already a failed value.
413-
// If current transfer status has already failed value, then it will not be changed.
414-
return common.Iffint32(startVal < 0, startVal, int32(status)), nil
412+
// If current transfer status has some completed value, then it will not be changed.
413+
return common.Iffint32(common.TransferStatus(startVal).StatusLocked(), startVal, int32(status)), nil
415414
})
416415
} else {
417416
(&jppt.atomicTransferStatus).AtomicStore(status)

0 commit comments

Comments
 (0)