feat: allow to rerun only failed tests (ui) (cypress-io#4886)

Co-authored-by: Tomás Teixeira <tomas.teixeira@tecnico.ulisboa.pt>

Author: tiago80673

packages/app/cypress/e2e/sidebar_navigation.cy.ts

@@ -119,9 +119,11 @@ describe('Sidebar Navigation', { viewportWidth: 1280 }, () => {
       cy.get('li p').contains('Re-run tests').should('be.visible')
       cy.get('li p').contains('Stop tests').should('be.visible')
       cy.get('li p').contains('Toggle specs list').should('be.visible')
+      cy.get('li p').contains('Re-run failed tests').should('be.visible')
       cy.get('li span').contains('r')
       cy.get('li span').contains('s')
       cy.get('li span').contains('f')
+      cy.get('li span').contains('t')
 
       // cy.percySnapshot() // TODO: restore when Percy CSS is fixed. See https://github.com/cypress-io/cypress/issues/23435
       cy.get('[aria-label="Close"]').click()
@@ -216,9 +218,11 @@ describe('Sidebar Navigation', { viewportWidth: 1280 }, () => {
       cy.get('li p').contains('Re-run tests').should('be.visible')
       cy.get('li p').contains('Stop tests').should('be.visible')
       cy.get('li p').contains('Toggle specs list').should('be.visible')
+      cy.get('li p').contains('Re-run failed tests').should('be.visible')
       cy.get('li span').contains('r')
       cy.get('li span').contains('s')
       cy.get('li span').contains('f')
+      cy.get('li span').contains('t')
       cy.get('[aria-label="Close"]').click()
       cy.findByText('Keyboard shortcuts').should('not.exist')
     })

packages/app/src/navigation/KeyboardBindingsModal.vue

@@ -56,5 +56,9 @@ const keyBindings = [
     key: ['f'],
     description: t('sidebar.keyboardShortcuts.toggle'),
   },
+  {
+    key: ['t'],
+    description: t('sidebar.keyboardShortcuts.failed'),
+  },
 ]
 </script>

packages/app/src/runner/event-manager.ts

@@ -99,6 +99,15 @@ export class EventManager {
       return this.rerunSpec()
     }
 
+    const rerunF = () => {
+      if (!this) {
+        // if the tests have been reloaded then there is nothing to rerun
+        return
+      }
+
+      return this.rerunFailedSpec()
+    }
+
     const connectionInfo: AddGlobalListenerOptions = {
       element: options.element,
       randomString: options.randomString,
@@ -214,6 +223,8 @@ export class EventManager {
 
     this.reporterBus.on('runner:restart', rerun)
 
+    this.reporterBus.on('runner:restart-failed', rerunF)
+
     const sendEventIfSnapshotProps = (testId, logId, event) => {
       if (!Cypress) return
 
@@ -857,6 +868,21 @@ export class EventManager {
     this.localBus.emit('restart')
   }
 
+  async rerunFailedSpec () {
+    if (!this || !Cypress) {
+      // if the tests have been reloaded then there is nothing to rerun
+      return
+    }
+
+    await this.resetReporter()
+
+    // this probably isn't 100% necessary since Cypress will fall out of scope
+    // but we want to be aggressive here and force GC early and often
+    Cypress.removeAllListeners()
+
+    this.localBus.emit('restart-failed')
+  }
+
   _interceptStudio (displayProps) {
     if (this.studioStore.isActive) {
       displayProps.hookId = this.studioStore.hookId

packages/app/src/runner/index.ts

@@ -411,7 +411,7 @@ async function initialize () {
  * 5. Setup the spec. This involves a few things, see the `runSpecCT` function's
  *    description for more information.
  */
-async function executeSpec (spec: SpecFile, isRerun: boolean = false) {
+async function executeSpec (spec: SpecFile, isRerun: boolean = false, skipPassed: boolean = false) {
   await teardownSpec(isRerun)
 
   const mobxRunnerStore = getMobxRunnerStore()
@@ -425,6 +425,9 @@ async function executeSpec (spec: SpecFile, isRerun: boolean = false) {
   // TODO: UNIFY-1318 - figure out how to manage window.config.
   const config = getRunnerConfigFromWindow()
 
+  // this lets Cypress driver skip passed tests
+  spec.skipPassed = skipPassed
+
   // this is how the Cypress driver knows which spec to run.
   config.spec = setSpecForDriver(spec)
 

packages/app/src/runner/useEventManager.ts

@@ -13,13 +13,13 @@ export function useEventManager () {
   const studioStore = useStudioStore()
   const router = useRouter()
 
-  function runSpec (isRerun: boolean = false) {
+  function runSpec (isRerun: boolean = false, skipPassed: boolean = false) {
     if (!specStore.activeSpec) {
       throw Error(`Cannot run spec when specStore.active spec is null or undefined!`)
     }
 
     autStore.setScriptError(null)
-    UnifiedRunnerAPI.executeSpec(specStore.activeSpec, isRerun)
+    UnifiedRunnerAPI.executeSpec(specStore.activeSpec, isRerun, skipPassed)
   }
 
   function initializeRunnerLifecycleEvents () {
@@ -33,6 +33,15 @@ export function useEventManager () {
       }
     })
 
+    eventManager.on('restart-failed', () => {
+      // If we get the event to restart but have already navigated away from the runner, don't execute the spec
+      if (specStore.activeSpec) {
+        const isRerun = true
+
+        runSpec(isRerun, true)
+      }
+    })
+
     eventManager.on('script:error', (err) => {
       autStore.setScriptError(err)
     })

packages/driver/cypress/e2e/issues/4886.cy.js

@@ -0,0 +1,89 @@
+/// <reference types="cypress" />
+
+import { flattenTestsFromSuites, loadPassedTests, savePassedTests, setSkipOnPassedTests } from '../../../src/cypress/runner'
+
+// Sample data for testing
+const sampleSuite = {
+  suites: [
+    {
+      tests: [
+        { id: 1, body: 'test1', state: 'passed' },
+        { id: 2, body: 'test2', state: 'failed' },
+      ],
+      suites: [
+        {
+          tests: [
+            { id: 3, body: 'test3', state: 'passed' },
+            { id: 4, body: 'test4', state: 'failed' },
+          ],
+        },
+      ],
+    },
+  ],
+}
+
+describe('Test Suite Processing Functions', () => {
+  beforeEach(() => {
+    cy.window().then((win) => {
+      win.passedTestsInfo = {}
+    })
+  })
+
+  it('should flatten all tests from nested suites', () => {
+    const result = flattenTestsFromSuites(sampleSuite)
+
+    expect(result).to.have.length(4)
+    expect(result.map((test) => test.id)).to.include.members([1, 2, 3, 4])
+  })
+
+  it('should return passed tests from window object', () => {
+    const testLocation = 'testPath'
+
+    cy.window().then((win) => {
+      win.passedTestsInfo[testLocation] = [{ id: 1, body: 'test1' }]
+
+      const result = loadPassedTests(testLocation)
+
+      expect(result).to.have.length(1)
+      expect(result[0].id).to.equal(1)
+    })
+  })
+
+  it('should return an empty array if no passed tests are found', () => {
+    const testLocation = 'testPath'
+
+    cy.window().then((win) => {
+      win.passedTestsInfo = {}
+
+      const result = loadPassedTests(testLocation)
+
+      expect(result).to.be.an('array').that.is.empty
+    })
+  })
+
+  it('should save passed tests and remove failed ones', () => {
+    const savedInfo = [{ id: 1, body: 'test1' }]
+    const testLocation = 'testPath'
+
+    savePassedTests(sampleSuite, savedInfo, testLocation)
+
+    cy.window().then((win) => {
+      const info = win.passedTestsInfo[testLocation]
+
+      expect(info).to.have.length(2)
+      expect(info.map((test) => test.id)).to.include.members([1, 3])
+    })
+  })
+
+  it('should set tests to pending if they were previously passed', () => {
+    const savedInfo = [{ id: 1, body: 'test1' }, { id: 3, body: 'test3' }]
+
+    setSkipOnPassedTests(sampleSuite, savedInfo)
+
+    const allTests = flattenTestsFromSuites(sampleSuite)
+    const pendingTests = allTests.filter((test) => test.pending === 'true')
+
+    expect(pendingTests).to.have.length(2)
+    expect(pendingTests.map((test) => test.id)).to.include.members([1, 3])
+  })
+})

packages/driver/src/cypress/runner.ts

@@ -361,6 +361,76 @@ const isLastSuite = (suite, tests) => {
   .value() === suite
 }
 
+export function flattenTestsFromSuites (data) {
+  let allTests = []
+
+  function extractTests (obj) {
+    if (obj.tests) {
+      allTests = allTests.concat(obj.tests)
+    }
+
+    if (obj.suites) {
+      obj.suites.forEach((suite) => extractTests(suite))
+    }
+  }
+
+  extractTests(data)
+
+  return allTests
+}
+
+// retrieve data from the last run to enable skipping
+export function loadPassedTests (testLocation) {
+  if (!window.passedTestsInfo) {
+    window.passedTestsInfo = {}
+  }
+
+  //TODO should be from a file and not from window
+  return window.passedTestsInfo[testLocation] ? window.passedTestsInfo[testLocation] : []
+  //return $utils.loadLastRunInfoFromFile(specsPath)
+}
+
+// this allows saving the info from the last run to enable skipping
+export function savePassedTests (suite, savedInfo, testLocation) {
+  //if the tests finished running the info is suite.suites[0]
+  //if the tests were stopped during execution the info is still on suite
+  let currentTests = flattenTestsFromSuites(suite)
+
+  function saveTest (test) {
+    if (!test) {
+      return
+    }
+
+    if (test.state === 'passed') {
+      if (!savedInfo.some((savedTest) => savedTest.id === test.id)) {
+        savedInfo.push({ 'id': test.id, 'body': test.body })
+      }
+    } else if (test.state === 'failed') {
+      if (savedInfo.some((savedTest) => savedTest.id === test.id)) {
+        savedInfo = savedInfo.filter((savedTest) => savedTest.id !== test.id)
+      }
+    }
+  }
+
+  currentTests.forEach(saveTest.bind(this))
+
+  //TODO should be from a file and not from window
+  window.passedTestsInfo[testLocation] = savedInfo
+  //$utils.saveLastRunInfoToFile(specsPath, savedInfo)
+}
+
+export function setSkipOnPassedTests (suite, savedInfo) {
+  let currentTests = flattenTestsFromSuites(suite)
+
+  function skipTest (test) {
+    if (savedInfo.some((savedTest) => savedTest.body === test.body)) {
+      test.pending = 'true'
+    }
+  }
+
+  currentTests.forEach(skipTest.bind(this))
+}
+
 // we are the last test that will run in the suite
 // if we're the last test in the tests array or
 // if we failed from a hook and that hook was 'before'
@@ -1217,9 +1287,15 @@ export default {
     let _uncaughtFn: (() => never) | null = null
     let _resumedAtTestIndex: number | null = null
     let _skipCollectingLogs = true
+    let _passedTests = []
+    let _shouldSkipPassedTests = false
     const _runner = mocha.getRunner()
 
+    _passedTests = loadPassedTests(Cypress.spec.absolute)
     _runner.suite = mocha.getRootSuite()
+    if (Cypress.spec.skipPassed) {
+      _shouldSkipPassedTests = true
+    }
 
     function isNotAlreadyRunTest (test) {
       return _resumedAtTestIndex == null || getTestIndexFromId(test.id) >= _resumedAtTestIndex
@@ -1583,6 +1659,10 @@ export default {
           mocha.createRootTest('An uncaught error was detected outside of a test', _uncaughtFn)
         }
 
+        if (_shouldSkipPassedTests) {
+          setSkipOnPassedTests(_runner.suite, _passedTests)
+        }
+
         return normalizeAll(
           _runner.suite,
           tests,
@@ -1905,6 +1985,7 @@ export default {
       },
 
       stop () {
+        savePassedTests(_runner.suite, _passedTests, Cypress.spec.absolute)
         if (_runner.stopped) {
           return
         }

packages/driver/src/cypress/utils.ts

@@ -3,6 +3,8 @@ import capitalize from 'underscore.string/capitalize'
 import methods from 'methods'
 import dayjs from 'dayjs'
 import $ from 'jquery'
+import * as fs from 'fs'
+import * as path from 'path'
 
 import $dom from '../dom'
 import $jquery from '../dom/jquery'
@@ -419,4 +421,43 @@ export default {
     // @ts-ignore
     return ret && _.isObject(ret) && 'then' in ret && _.isFunction(ret.then) && 'catch' in ret && _.isFunction(ret.catch)
   },
+
+  checkFileExists (file) {
+    return fs.promises.access(file, fs.constants.F_OK)
+    .then(() => true)
+    .catch(() => false)
+  },
+
+  loadLastRunInfoFromFile (pathF) {
+    const directory = path.dirname('/')
+    const fileBaseName = path.basename(pathF, path.extname(pathF))
+    const filePath = path.join(directory, fileBaseName+'-passed.txt')
+
+    try {
+      if (fs.existsSync(filePath)) {
+        fs.promises.readFile(filePath, 'utf8').then((data) => {
+          return JSON.parse(data)
+        })
+      }
+
+      return []
+    } catch (error) {
+      // do smth
+      return []
+    }
+  },
+
+  async saveLastRunInfoToFile (pathF, testInfo) {
+    const directory = path.dirname('/')
+    const fileBaseName = path.basename(pathF, path.extname(pathF))
+    const filePath = path.join(directory, fileBaseName+'-passed')
+
+    try {
+      const data = JSON.stringify(testInfo, null, 2)
+
+      fs.promises.writeFile(filePath, data, { encoding: 'utf8', flag: 'w' })
+    } catch (error) {
+      // do smth
+    }
+  },
 }