feat: run filtered debug tests in runner (#25265)

Co-authored-by: Emily Rohrbough <emilyrohrbough@users.noreply.github.com>
Co-authored-by: Mark Noonan <mark@cypress.io>
Co-authored-by: Mike Plummer <mikep@cypress.io>
Closes #24855

Author: ZachJW34

packages/app/cypress/e2e/runner/cloud-debug-filter.cy.ts

@@ -0,0 +1,139 @@
+describe('cloud debug test filtering', () => {
+  beforeEach(() => {
+    cy.scaffoldProject('cloud-debug-filter')
+    cy.openProject('cloud-debug-filter')
+    cy.startAppServer('e2e')
+  })
+
+  it('works with nested suites', () => {
+    cy.visitApp(`specs/runner?file=cypress/e2e/test.cy.js`)
+
+    cy.waitForSpecToFinish()
+
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = ['t2']
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/test.cy.js&runId=123`)
+    cy.waitForSpecToFinish({ passCount: 0, failCount: 1 })
+
+    cy.get('.runnable-title').contains('t2')
+
+    cy.get('.debug-dismiss').contains('1 / 4 tests').click()
+    cy.waitForSpecToFinish({ passCount: 2, failCount: 2 })
+
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = ['s1 t4']
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/test.cy.js&runId=123`)
+    cy.waitForSpecToFinish({ passCount: 0, failCount: 1 })
+
+    cy.get('.runnable-title').contains('t4')
+  })
+
+  it('wraps filter UI with large number of tests', () => {
+    cy.visitApp(`specs/runner?file=cypress/e2e/lots-of-tests.cy.js`)
+
+    cy.get('[data-cy="reporter-panel"]').as('reporterPanel')
+
+    cy.waitForSpecToFinish()
+
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = ['test1']
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/lots-of-tests.cy.js&runId=123`)
+    cy.waitForSpecToFinish({ passCount: 50 })
+
+    cy.get('@reporterPanel').then((el) => el.width(500))
+    cy.get('@reporterPanel').percySnapshot('wide')
+
+    cy.get('@reporterPanel').then((el) => el.width(350))
+    cy.get('@reporterPanel').percySnapshot('medium')
+
+    cy.get('@reporterPanel').then((el) => el.width(250))
+    cy.get('@reporterPanel').percySnapshot('narrow')
+
+    cy.get('@reporterPanel').then((el) => el.width(150))
+    cy.get('@reporterPanel').percySnapshot('skinny')
+  })
+
+  it('works with skips and onlys', () => {
+    cy.visitApp(`specs/runner?file=cypress/e2e/skip-and-only.cy.js`)
+
+    cy.waitForSpecToFinish({ passCount: 0, failCount: 1 })
+
+    // .only is respected
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = ['t1', 't3']
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/skip-and-only.cy.js&runId=123`)
+    cy.waitForSpecToFinish({ passCount: 0, failCount: 1 })
+
+    cy.get('.runnable-title').contains('t1')
+
+    cy.get('.debug-dismiss').click().waitForSpecToFinish()
+
+    // .only is ignored as it is not in set of filtered tests
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = ['t3']
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/skip-and-only.cy.js&runId=123`)
+    cy.waitForSpecToFinish({ passCount: 0, failCount: 1 })
+
+    cy.get('.runnable-title').contains('t3')
+
+    cy.get('.debug-dismiss').click().waitForSpecToFinish()
+
+    // .skip is respected
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = ['t2', 't3']
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/skip-and-only.cy.js&runId=123`)
+    cy.waitForSpecToFinish({ passCount: 0, failCount: 1, pendingCount: 1 })
+    cy.get('.runnable-title').first().contains('t2')
+    cy.get('.runnable-title').last().contains('t3')
+
+    cy.get('.debug-dismiss').contains('2 / 4 tests').click().waitForSpecToFinish()
+
+    // suite.only is respected
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = ['t3', 's1 t4']
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/skip-and-only.cy.js&runId=123`)
+    cy.waitForSpecToFinish({ passCount: 0, failCount: 1 })
+    cy.get('.runnable-title').contains('t4')
+  })
+
+  it('works with browser filter', () => {
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = ['t1', 's1 t2']
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/browsers.cy.js&runId=123`)
+
+    cy.get('.runnable-title').eq(0).contains('t1 (skipped due to browser)')
+    cy.get('.runnable-title').eq(1).contains('s1 (skipped due to browser)')
+    cy.get('.runnable-title').eq(2).contains('t2')
+  })
+
+  it('filter is maintained across cross-domain reinitialization', () => {
+    cy.visitApp(`specs/runner?file=cypress/e2e/domain-change.cy.js`)
+
+    cy.get('[data-cy="reporter-panel"]').as('reporterPanel')
+
+    cy.waitForSpecToFinish()
+
+    cy.withCtx((ctx) => {
+      ctx.coreData.cloud.testsForRunResults = ['t2', 't3']
+    })
+
+    cy.visitApp(`specs/runner?file=cypress/e2e/domain-change.cy.js&runId=123`)
+    cy.waitForSpecToFinish({ failCount: 2 })
+  })
+})

packages/app/src/runner/event-manager-types.ts

@@ -19,6 +19,7 @@ export type LocalBusEventMap = {
   'before:screenshot': BeforeScreenshot
   'after:screenshot': undefined
   'open:file': FileDetails
+  'testFilter:cloudDebug:dismiss': undefined
 }
 
 export interface StudioSavePayload {
@@ -46,6 +47,9 @@ export type LocalBusEmitsMap = {
   'reporter:log:add': CommandLog
   'reporter:log:remove': CommandLog
   'reporter:log:state:changed': CommandLog
+
+  // Test filter events
+  'testFilter:cloudDebug:dismiss': undefined
 }
 
 export type SocketToDriverMap = {

packages/app/src/runner/event-manager.ts

@@ -10,6 +10,7 @@ import { logger } from './logger'
 import type { Socket } from '@packages/socket/lib/browser'
 import { automation, useRunnerUiStore } from '../store'
 import { useScreenshotStore } from '../store/screenshot-store'
+import { useSpecStore } from '../store/specs-store'
 import { useStudioStore } from '../store/studio-store'
 import { getAutIframeModel } from '.'
 import { handlePausing } from './events/pausing'
@@ -56,6 +57,7 @@ export class EventManager {
   cypressInCypressMochaEvents: CypressInCypressMochaEvent[] = []
   // Used for testing the experimentalSingleTabRunMode experiment. Ensures AUT is correctly destroyed between specs.
   ws: Socket
+  specStore: ReturnType<typeof useSpecStore>
   studioStore: ReturnType<typeof useStudioStore>
 
   constructor (
@@ -69,6 +71,7 @@ export class EventManager {
   ) {
     this.selectorPlaygroundModel = selectorPlaygroundModel
     this.ws = ws
+    this.specStore = useSpecStore()
     this.studioStore = useStudioStore()
   }
 
@@ -236,6 +239,10 @@ export class EventManager {
       this.saveState(state)
     })
 
+    this.reporterBus.on('testFilter:cloudDebug:dismiss', () => {
+      this.emit('testFilter:cloudDebug:dismiss', undefined)
+    })
+
     this.reporterBus.on('clear:all:sessions', () => {
       if (!Cypress) return
 
@@ -379,6 +386,8 @@ export class EventManager {
   initialize ($autIframe: JQuery<HTMLIFrameElement>, config: Record<string, any>) {
     performance.mark('initialize-start')
 
+    const testFilter = this.specStore.testFilter
+
     return Cypress.initialize({
       $autIframe,
       onSpecReady: () => {
@@ -394,7 +403,7 @@ export class EventManager {
 
           this.studioStore.initialize(config, runState)
 
-          const runnables = Cypress.runner.normalizeAll(runState.tests, hideCommandLog)
+          const runnables = Cypress.runner.normalizeAll(runState.tests, hideCommandLog, testFilter)
 
           const run = () => {
             performance.mark('initialize-end')

packages/app/src/runner/index.ts

@@ -24,7 +24,7 @@ import { AutIframe } from './aut-iframe'
 import { EventManager } from './event-manager'
 import { createWebsocket as createWebsocketIo } from '@packages/socket/lib/browser'
 import { decodeBase64Unicode } from '@packages/frontend-shared/src/utils/base64'
-import type { AutomationElementId } from '@packages/types/src'
+import type { AutomationElementId } from '@packages/types'
 import { useSnapshotStore } from './snapshot-store'
 import { useStudioStore } from '../store/studio-store'
 
@@ -374,7 +374,7 @@ async function initialize () {
  * 2. Reset the Reporter. We use the same instance of the Reporter,
  *    but reset the internal state each time we run a spec.
  *
- * 3. Teardown spec. This does a few things, primaily stopping the current
+ * 3. Teardown spec. This does a few things, primarily stopping the current
  *    spec run, which involves stopping the driver and runner.
  *
  * 4. Force the Reporter to re-render with the new spec we are executed.

packages/app/src/runner/reporter.ts

@@ -1,4 +1,4 @@
-import { getMobxRunnerStore, MobxRunnerStore } from '../store'
+import { getMobxRunnerStore, MobxRunnerStore, useSpecStore } from '../store'
 import { getReporterElement } from './utils'
 import { getEventManager, getRunnerConfigFromWindow } from '.'
 import type { EventManager } from './event-manager'
@@ -38,6 +38,7 @@ function renderReporter (
   eventManager: EventManager,
 ) {
   const runnerUiStore = useRunnerUiStore()
+  const specsStore = useSpecStore()
 
   const config = getRunnerConfigFromWindow()
 
@@ -51,6 +52,7 @@ function renderReporter (
     // Studio can only be enabled for e2e testing
     studioEnabled: window.__CYPRESS_TESTING_TYPE__ === 'e2e' && config.experimentalStudio,
     runnerStore: store,
+    testFilter: specsStore.testFilter,
   })
 
   window.UnifiedRunner.ReactDOM.render(reporter, root)

packages/app/src/runner/unifiedRunner.ts

@@ -3,12 +3,23 @@ import { getAutIframeModel, UnifiedRunnerAPI } from '../runner'
 import { useSpecStore } from '../store'
 import { useSelectorPlaygroundStore } from '../store/selector-playground-store'
 import { RUN_ALL_SPECS, RUN_ALL_SPECS_KEY, SpecFile } from '@packages/types/src'
-import { useRoute } from 'vue-router'
+import { LocationQuery, useRoute } from 'vue-router'
 import { getPathForPlatform } from '../paths'
+import { isEqual } from 'lodash'
+import { gql, useMutation } from '@urql/vue'
+import { TestsForRunDocument } from '../generated/graphql'
+
+gql`
+mutation TestsForRun ($runId: String!) {
+  testsForRun (runId: $runId)
+}
+`
 
 const initialized = ref(false)
 
 export function useUnifiedRunner () {
+  let prevQuery: LocationQuery
+
   onMounted(async () => {
     await UnifiedRunnerAPI.initialize()
     initialized.value = true
@@ -25,8 +36,9 @@ export function useUnifiedRunner () {
       const specStore = useSpecStore()
       const route = useRoute()
       const selectorPlaygroundStore = useSelectorPlaygroundStore()
+      const testsForRunMutation = useMutation(TestsForRunDocument)
 
-      watchEffect(() => {
+      watchEffect(async () => {
         const queryFile = getPathForPlatform(route.query.file as string)
 
         if (!queryFile) {
@@ -35,17 +47,32 @@ export function useUnifiedRunner () {
           return
         }
 
-        if (queryFile === RUN_ALL_SPECS_KEY) {
-          return specStore.setActiveSpec(RUN_ALL_SPECS)
+        const activeSpecInSpecsList = queryFile === RUN_ALL_SPECS_KEY
+          ? RUN_ALL_SPECS
+          : specs.value.find((x) => x.relative === queryFile)
+
+        if (isEqual(route.query, prevQuery) && isEqual(activeSpecInSpecsList, specStore.activeSpec)) {
+          return
         }
 
-        const activeSpecInSpecsList = specs.value.find((x) => x.relative === queryFile)
+        prevQuery = route.query
 
-        if (activeSpecInSpecsList && specStore.activeSpec?.relative !== activeSpecInSpecsList.relative) {
-          specStore.setActiveSpec(activeSpecInSpecsList)
-        } else if (!activeSpecInSpecsList) {
-          specStore.setActiveSpec(null)
+        if (!activeSpecInSpecsList) {
+          return specStore.setActiveSpec(null)
         }
+
+        if (route.query.runId) {
+          const res = await testsForRunMutation.executeMutation({ runId: route.query.runId as string })
+
+          specStore.setTestFilter(res.data?.testsForRun?.length ? res.data.testsForRun : undefined)
+        } else {
+          specStore.setTestFilter(undefined)
+        }
+
+        // Either there is a new spec or the runId has changed. Returning a new object will kick off a new run.
+        // A user could be looking at a given run when a new run is kicked off by CI or another user.
+        // They can view the newer run using an in-UI transition which updates the runId and kicks off new queries.
+        specStore.setActiveSpec({ ...activeSpecInSpecsList })
       })
 
       watch(() => getPathForPlatform(route.query.file as string), () => {

packages/app/src/runner/useEventManager.ts

@@ -1,4 +1,5 @@
 import { watch } from 'vue'
+import { useRouter } from 'vue-router'
 import { addCrossOriginIframe, getAutIframeModel, getEventManager, UnifiedRunnerAPI } from '.'
 import { useAutStore, useSpecStore } from '../store'
 import { useStudioStore } from '../store/studio-store'
@@ -10,6 +11,7 @@ export function useEventManager () {
   const autStore = useAutStore()
   const specStore = useSpecStore()
   const studioStore = useStudioStore()
+  const router = useRouter()
 
   function runSpec (isRerun: boolean = false) {
     if (!specStore.activeSpec) {
@@ -58,6 +60,15 @@ export function useEventManager () {
     })
 
     eventManager.on('expect:origin', addCrossOriginIframe)
+
+    eventManager.on('testFilter:cloudDebug:dismiss', () => {
+      const currentRoute = router.currentRoute.value
+
+      const { runId, ...query } = currentRoute.query
+
+      // Delete runId from query which will remove the test filter and trigger a rerun
+      router.replace({ ...currentRoute, query })
+    })
   }
 
   const startSpecWatcher = () => {

packages/app/src/store/specs-store.ts

@@ -1,18 +1,19 @@
-import type { SpecFile } from '@packages/types/src'
+import type { SpecFile, TestFilter } from '@packages/types/src'
 import { defineStore } from 'pinia'
 
 export interface SpecState {
   activeSpec: SpecFile | null | undefined
   specFilter?: string
+  testFilter: TestFilter
 }
 
 export const useSpecStore = defineStore({
   id: 'spec',
-
   state (): SpecState {
     return {
       activeSpec: undefined,
       specFilter: undefined,
+      testFilter: undefined,
     }
   },
 
@@ -23,5 +24,8 @@ export const useSpecStore = defineStore({
     setSpecFilter (filter: string) {
       this.specFilter = filter
     },
+    setTestFilter (filter: SpecState['testFilter']) {
+      this.testFilter = filter
+    },
   },
 })