Skip to content

Commit 067e4fa

Browse files
jordanpowell88astone123
jordanpowell88
and
astone123
authored
test: fix flaky tests (#26505)
* test(launchpad): fix flaky tests Closes #23153 Closes #23154 --------- Co-authored-by: astone123 <adams@cypress.io>
1 parent 6209b91 commit 067e4fa

8 files changed

+52
-46
lines changed

‎.circleci/workflows.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ windowsWorkflowFilters: &windows-workflow-filters
6969
or:
7070
- equal: [ develop, << pipeline.git.branch >> ]
7171
# use the following branch as well to ensure that v8 snapshot cache updates are fully tested
72-
- equal: [ 'lmiller/fixing-vite-windows', << pipeline.git.branch >> ]
72+
- equal: [ 'windows-flake', << pipeline.git.branch >> ]
7373
- equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ]
7474
- matches:
7575
pattern: /^release\/\d+\.\d+\.\d+$/

‎packages/app/cypress/e2e/runs.cy.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,7 @@ describe('App: Runs', { viewportWidth: 1200 }, () => {
323323
moveToRunsPage()
324324
cy.findByText(defaultMessages.runs.connect.buttonProject).click()
325325
cy.contains('button', defaultMessages.runs.connect.modal.selectProject.createProject).click()
326-
cy.findByText(defaultMessages.runs.connectSuccessAlert.title).should('be.visible')
326+
cy.findByText(defaultMessages.runs.connectSuccessAlert.title, { timeout: 10000 }).should('be.visible')
327327

328328
cy.withCtx(async (ctx) => {
329329
const config = await ctx.project.getConfig()

‎packages/launchpad/cypress/e2e/config-files-error-handling.cy.ts

+23-23
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ describe('Config files error handling', () => {
3131

3232
cy.findByRole('button', { name: 'Try again' }).click()
3333

34-
cy.get('h1').should('contain', 'Welcome to Cypress')
34+
cy.contains('h1', 'Welcome to Cypress', { timeout: 10000 })
3535
})
3636

3737
it('shows the upgrade screen if there is a legacy config file', () => {
@@ -70,7 +70,7 @@ describe('Config files error handling', () => {
7070

7171
cy.findByRole('button', { name: 'Try again' }).click()
7272

73-
cy.get('h1').should('contain', 'Welcome to Cypress')
73+
cy.contains('h1', 'Welcome to Cypress', { timeout: 10000 })
7474
})
7575

7676
it('handles deprecated config fields', () => {
@@ -92,7 +92,7 @@ describe('Config files error handling', () => {
9292

9393
cy.findByRole('button', { name: 'Try again' }).click()
9494

95-
cy.get('h1').should('contain', 'Choose a browser')
95+
cy.contains('h1', 'Choose a browser', { timeout: 10000 })
9696
})
9797
})
9898

@@ -102,7 +102,7 @@ describe('Launchpad: Error System Tests', () => {
102102
cy.openProject('plugins-root-sync-error', ['--e2e'])
103103
cy.visitLaunchpad()
104104
cy.skipWelcome()
105-
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
105+
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
106106
expectStackToBe('open')
107107
})
108108

@@ -111,7 +111,7 @@ describe('Launchpad: Error System Tests', () => {
111111
cy.openProject('plugins-root-syntax-error', ['--e2e'])
112112
cy.visitLaunchpad()
113113
cy.skipWelcome()
114-
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
114+
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
115115
expectStackToBe('open')
116116
})
117117

@@ -120,7 +120,7 @@ describe('Launchpad: Error System Tests', () => {
120120
cy.openProject('plugins-root-async-error', ['--e2e'])
121121
cy.visitLaunchpad()
122122
cy.skipWelcome()
123-
cy.contains('h1', 'Unexpected Error')
123+
cy.contains('h1', 'Unexpected Error', { timeout: 10000 })
124124
expectStackToBe('open')
125125
})
126126

@@ -129,7 +129,7 @@ describe('Launchpad: Error System Tests', () => {
129129
cy.openProject('plugins-function-sync-error', ['--e2e'])
130130
cy.visitLaunchpad()
131131
cy.skipWelcome()
132-
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
132+
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
133133
expectStackToBe('open')
134134
})
135135

@@ -138,7 +138,7 @@ describe('Launchpad: Error System Tests', () => {
138138
cy.openProject('config-with-invalid-browser', ['--e2e'])
139139
cy.visitLaunchpad()
140140
cy.skipWelcome()
141-
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
141+
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
142142
expectStackToBe('closed')
143143
})
144144

@@ -147,7 +147,7 @@ describe('Launchpad: Error System Tests', () => {
147147
cy.openProject('plugins-function-sync-error', ['--e2e'])
148148
cy.visitLaunchpad()
149149
cy.skipWelcome()
150-
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
150+
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
151151
expectStackToBe('open')
152152
})
153153

@@ -156,7 +156,7 @@ describe('Launchpad: Error System Tests', () => {
156156
cy.openProject('config-with-ts-syntax-error')
157157
cy.visitLaunchpad()
158158
cy.skipWelcome()
159-
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
159+
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
160160
cy.percySnapshot()
161161

162162
cy.withCtx(async (ctx) => {
@@ -165,7 +165,7 @@ describe('Launchpad: Error System Tests', () => {
165165

166166
cy.findByRole('button', { name: 'Try again' }).click()
167167

168-
cy.get('h1').should('contain', 'Welcome to Cypress')
168+
cy.contains('h1', 'Welcome to Cypress', { timeout: 10000 })
169169
})
170170

171171
it(`clears the error correctly after first 'try again' attempt`, () => {
@@ -174,31 +174,31 @@ describe('Launchpad: Error System Tests', () => {
174174
cy.openProject('config-with-ts-syntax-error')
175175
cy.visitLaunchpad()
176176
cy.skipWelcome()
177-
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
177+
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
178178

179179
// Try again while the config is still invalid
180180
cy.findByRole('button', { name: 'Try again' }).click()
181181

182182
cy.wait('@resetErrorsAndLoadConfig')
183183

184184
// Wait until config error is on screen again
185-
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
185+
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
186186

187187
cy.withCtx(async (ctx) => {
188188
await ctx.actions.file.writeFileInProject('cypress.config.ts', 'export default { e2e: { supportFile: false } }')
189189
})
190190

191191
cy.findByRole('button', { name: 'Try again' }).click()
192192

193-
cy.get('h1').should('contain', 'Welcome to Cypress')
193+
cy.contains('h1', 'Welcome to Cypress')
194194
})
195195

196196
it('shows correct user file instead of node file', () => {
197197
cy.scaffoldProject('config-with-import-error')
198198
cy.openProject('config-with-import-error')
199199
cy.visitLaunchpad()
200200
cy.skipWelcome()
201-
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
201+
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
202202
cy.percySnapshot()
203203

204204
cy.get('[data-testid="error-code-frame"]').should('contain', 'cypress.config.js:3:23')
@@ -209,7 +209,7 @@ describe('Launchpad: Error System Tests', () => {
209209
cy.openProject('config-with-ts-module-error')
210210
cy.visitLaunchpad()
211211
cy.skipWelcome()
212-
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
212+
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
213213
cy.percySnapshot()
214214

215215
cy.get('[data-testid="error-code-frame"]').should('contain', 'cypress.config.ts:6:10')
@@ -223,7 +223,7 @@ describe('setupNodeEvents', () => {
223223
cy.visitLaunchpad()
224224
cy.skipWelcome()
225225
cy.findByText('E2E Testing').click()
226-
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
226+
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
227227
cy.percySnapshot()
228228
})
229229

@@ -233,7 +233,7 @@ describe('setupNodeEvents', () => {
233233
cy.visitLaunchpad()
234234
cy.skipWelcome()
235235
cy.findByText('E2E Testing').click()
236-
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
236+
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
237237
cy.percySnapshot()
238238

239239
cy.get('[data-cy="alert-body"]').should('contain', 'integrationFolder')
@@ -245,7 +245,7 @@ describe('setupNodeEvents', () => {
245245
cy.visitLaunchpad()
246246
cy.skipWelcome()
247247
cy.findByText('E2E Testing').click()
248-
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
248+
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
249249
cy.percySnapshot()
250250
})
251251

@@ -279,7 +279,7 @@ describe('setupNodeEvents', () => {
279279

280280
cy.findByRole('button', { name: 'Try again' }).click()
281281

282-
cy.get('h1').should('contain', 'Choose a browser')
282+
cy.contains('h1', 'Choose a browser', { timeout: 10000 })
283283
})
284284

285285
it('handles multiple config errors and then recovers', () => {
@@ -293,7 +293,7 @@ describe('setupNodeEvents', () => {
293293

294294
cy.visitLaunchpad()
295295
cy.skipWelcome()
296-
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
296+
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
297297
cy.get('[data-cy="alert-body"]').should('contain', 'Expected baseUrl to be a fully qualified URL')
298298

299299
cy.withCtx(async (ctx) => {
@@ -302,15 +302,15 @@ describe('setupNodeEvents', () => {
302302

303303
cy.findByRole('button', { name: 'Try again' }).click()
304304
cy.get('[data-cy-testingType=e2e]').click()
305-
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle)
305+
cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 })
306306
cy.get('[data-cy="alert-body"]').should('contain', 'The baseUrl configuration option is now invalid when set from the root of the config object')
307307

308308
cy.withCtx(async (ctx) => {
309309
await ctx.actions.file.writeFileInProject('cypress.config.js', `module.exports = { e2e: { baseUrl: 'http://localhost:3000', supportFile: false } }`)
310310
})
311311

312312
cy.findByRole('button', { name: 'Try again' }).click()
313-
cy.get('h1').should('contain', 'Choose a browser')
313+
cy.contains('h1', 'Choose a browser', { timeout: 10000 })
314314
cy.get('[data-cy="alert"]').should('contain', 'Warning: Cannot Connect Base Url Warning')
315315
})
316316

‎packages/launchpad/cypress/e2e/config-warning.cy.ts

+5-6
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ describe('component testing dependency warnings', () => {
232232
cy.skipWelcome()
233233
cy.contains('component-testing-outdated-dependencies').click()
234234
cy.get('[data-cy="warning-alert"]').should('not.exist')
235-
cy.get('a').contains('Projects').click()
235+
cy.contains('a', 'Projects').click()
236236
cy.get('[data-cy-testingtype="component"]').click()
237237
cy.get('[data-cy="warning-alert"]', { timeout: 12000 }).should('exist')
238238
.should('contain.text', 'Warning: Component Testing Mismatched Dependencies')
@@ -243,17 +243,16 @@ describe('component testing dependency warnings', () => {
243243
cy.get('.warning-markdown').find('li').should('have.length', 3)
244244
})
245245

246-
// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23154
247-
it('warns against outdated @vue/cli dependency', { retries: 15 }, () => {
246+
it('warns against outdated @vue/cli dependency', () => {
248247
cy.scaffoldProject('outdated-deps-vuecli3')
249248
cy.addProject('outdated-deps-vuecli3')
250249
cy.openGlobalMode()
251250
cy.visitLaunchpad()
252251
cy.skipWelcome()
253252
cy.contains('outdated-deps-vuecli3').click()
254253
cy.get('[data-cy="warning-alert"]').should('not.exist')
255-
cy.get('a').contains('Projects').click()
256-
cy.get('[data-cy-testingtype="component"]').click()
254+
cy.contains('a', 'Projects').click()
255+
cy.get('[data-cy-testingtype="component"]', { timeout: 12000 }).click()
257256
cy.get('[data-cy="warning-alert"]', { timeout: 12000 }).should('exist')
258257
.should('contain.text', 'Warning: Component Testing Mismatched Dependencies')
259258
.should('contain.text', '@vue/cli-service. Expected ^=4.0.0 || ^=5.0.0, found 3.12.1.')
@@ -271,7 +270,7 @@ describe('component testing dependency warnings', () => {
271270
cy.contains('vueclivue3-configured').click()
272271
cy.get('[data-cy="warning-alert"]').should('not.exist')
273272
cy.get('a').contains('Projects').click()
274-
cy.get('[data-cy-testingtype="component"]').click()
273+
cy.get('[data-cy-testingtype="component"]', { timeout: 10000 }).click()
275274

276275
// Wait until launch browser screen and assert warning does not exist
277276
cy.contains('Choose a browser', { timeout: 12000 })

‎packages/launchpad/cypress/e2e/migration.cy.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ function finishMigrationAndContinue () {
5959
}
6060

6161
function checkOutcome () {
62-
cy.contains('Welcome to Cypress!').should('be.visible')
62+
cy.contains('Welcome to Cypress!', { timeout: 10000 }).should('be.visible')
6363
}
6464

6565
function runAutoRename () {
@@ -95,7 +95,7 @@ describe('global mode', () => {
9595
cy.contains('migration-e2e-export-default').click()
9696

9797
// rename integration->e2e
98-
cy.get(renameAutoStep).should('exist')
98+
cy.get(renameAutoStep, { timeout: 10000 }).should('exist')
9999
cy.get(renameManualStep).should('not.exist')
100100

101101
// cypress/support/index.ts -> cypress/support/e2e.ts

‎packages/launchpad/cypress/e2e/project-setup.cy.ts

+7-8
Original file line numberDiff line numberDiff line change
@@ -556,8 +556,7 @@ describe('Launchpad: Setup Project', () => {
556556
})
557557
})
558558

559-
// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23153
560-
describe('Command for package managers', { retries: 15 }, () => {
559+
describe('Command for package managers', () => {
561560
it('makes the right command for yarn', () => {
562561
scaffoldAndOpenProject('pristine-yarn')
563562

@@ -566,7 +565,7 @@ describe('Launchpad: Setup Project', () => {
566565
cy.get('[data-cy-testingtype="component"]').click()
567566
cy.get('[data-testid="select-framework"]').click()
568567
cy.findByText('Create React App').click()
569-
cy.findByText('Next step').click()
568+
cy.contains('button', 'Next step').should('not.be.disabled').click()
570569
cy.findByDisplayValue('yarn add -D react-scripts react-dom react').should('be.visible')
571570
})
572571

@@ -578,7 +577,7 @@ describe('Launchpad: Setup Project', () => {
578577
cy.get('[data-cy-testingtype="component"]').click()
579578
cy.get('[data-testid="select-framework"]').click()
580579
cy.findByText('Create React App').click()
581-
cy.findByText('Next step').click()
580+
cy.contains('button', 'Next step').should('not.be.disabled').click()
582581
cy.findByDisplayValue('pnpm install -D react-scripts react-dom react')
583582
})
584583

@@ -588,9 +587,9 @@ describe('Launchpad: Setup Project', () => {
588587
cy.visitLaunchpad()
589588

590589
cy.get('[data-cy-testingtype="component"]').click()
591-
cy.get('button').should('be.visible').contains('Vue.js 3(detected)')
592-
cy.get('button').should('be.visible').contains('Vite(detected)')
593-
cy.findByText('Next step').click()
590+
cy.contains('button', 'Vue.js 3(detected)').should('be.visible')
591+
cy.contains('button', 'Vite(detected)').should('be.visible')
592+
cy.contains('button', 'Next step').should('not.be.disabled').click()
594593
cy.findByTestId('alert').contains(`You've successfully installed all required dependencies.`)
595594
})
596595

@@ -602,7 +601,7 @@ describe('Launchpad: Setup Project', () => {
602601
cy.get('[data-cy-testingtype="component"]').click()
603602
cy.get('[data-testid="select-framework"]').click()
604603
cy.findByText('Create React App').click()
605-
cy.findByText('Next step').click()
604+
cy.contains('button', 'Next step').should('not.be.disabled').click()
606605
cy.findByDisplayValue('npm install -D react-scripts react-dom react')
607606
})
608607
})

‎packages/launchpad/cypress/e2e/scaffold-project.cy.ts

+9-1
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,10 @@ describe('scaffolding new projects', { defaultCommandTimeout: 7000 }, () => {
168168
})
169169

170170
it('generates valid config file for pristine project without cypress installed', () => {
171+
cy.intercept('mutation-ScaffoldedFiles_completeSetup').as('mutationScaffoldedFiles')
172+
cy.intercept('query-MainLaunchpadQuery').as('mainLaunchpadQuery')
173+
cy.intercept('query-HeaderBar_HeaderBarQuery').as('headerBarQuery')
174+
cy.intercept('query-CloudViewerAndProject_RequiredData').as('cloudViewerAndProjectRequiredData')
171175
cy.scaffoldProject('pristine')
172176
cy.openProject('pristine')
173177
cy.withCtx((ctx) => ctx.currentProject).then((currentProject) => {
@@ -178,6 +182,10 @@ describe('scaffolding new projects', { defaultCommandTimeout: 7000 }, () => {
178182
cy.skipWelcome()
179183
cy.contains('button', cy.i18n.testingType.e2e.name).click()
180184
cy.contains('button', cy.i18n.setupPage.step.continue).click()
181-
cy.contains('h1', cy.i18n.setupPage.testingCard.chooseABrowser).should('be.visible')
185+
cy.wait('@mutationScaffoldedFiles')
186+
cy.wait('@mainLaunchpadQuery')
187+
cy.wait('@headerBarQuery')
188+
cy.wait('@cloudViewerAndProjectRequiredData')
189+
cy.get('h1').contains(cy.i18n.setupPage.testingCard.chooseABrowser).should('be.visible')
182190
})
183191
})

‎packages/launchpad/cypress/e2e/top-nav-launchpad.cy.ts

+4-4
Original file line numberDiff line numberDiff line change
@@ -366,12 +366,12 @@ describe('Launchpad Top Nav Workflows', () => {
366366
options.sinon.stub(ctx._apis.authApi, 'logIn').callsFake(async (onMessage) => {
367367
setTimeout(() => {
368368
onMessage({ browserOpened: true })
369-
}, 500)
369+
}, 2000)
370370

371371
return new Promise((resolve) => {
372372
setTimeout(() => {
373373
resolve(options.user)
374-
}, 2000)
374+
}, 3000)
375375
})
376376
})
377377
}, { user })
@@ -687,7 +687,7 @@ describe('Launchpad Top Nav Workflows', () => {
687687

688688
cy.get('[data-cy="project-card"]').click()
689689

690-
cy.contains('E2E Testing').click()
690+
cy.contains('E2E Testing', { timeout: 10000 }).click()
691691

692692
mockLogInActionsForUser(mockUser)
693693
logIn({ expectedNextStepText: 'Continue', displayName: mockUser.name })
@@ -707,7 +707,7 @@ describe('Launchpad Top Nav Workflows', () => {
707707

708708
cy.get('[data-cy="project-card"]').click()
709709

710-
cy.contains('E2E Testing').click()
710+
cy.contains('E2E Testing', { timeout: 10000 }).click()
711711

712712
mockLogInActionsForUser(mockUser)
713713
logIn({ expectedNextStepText: 'Connect project', displayName: mockUser.name })

0 commit comments

Comments
 (0)