fix(server): use run#run instead of run#ready to replicate e2e exit behavior in CT (#17894)

Co-authored-by: Zach Bloomquist <git@chary.us>

Author: lmiller1990

circle.yml

@@ -373,6 +373,7 @@ commands:
             PERCY_ENABLE=${PERCY_TOKEN:-0} \
             PERCY_PARALLEL_TOTAL=-1 \
             $cmd yarn workspace @packages/runner cypress:run --record --parallel --group runner-integration-<<parameters.browser>> --browser <<parameters.browser>>
+      - verify-mocha-results
       - store_test_results:
           path: /tmp/cypress
       - store_artifacts:
@@ -407,6 +408,7 @@ commands:
             else
               echo "skipping percy screenshots uploading"
             fi
+      - verify-mocha-results
       - store_test_results:
           path: /tmp/cypress
       - store_artifacts:

packages/runner-ct/cypress.json

@@ -3,5 +3,9 @@
   "video": true,
   "env": {
     "reactDevtools": true
+  },
+  "reporter": "../../node_modules/cypress-multi-reporters/index.js",
+  "reporterOptions": {
+    "configFile": "../../mocha-reporter-config.json"
   }
 }

packages/runner/cypress.json

@@ -4,5 +4,9 @@
   "retries": {
     "runMode": 2,
     "openMode": 0
+  },
+  "reporter": "../../node_modules/cypress-multi-reporters/index.js",
+  "reporterOptions": {
+    "configFile": "../../mocha-reporter-config.json"
   }
 }

packages/server/lib/modes/run-ct.js

@@ -17,10 +17,7 @@ const run = (options) => {
     })
   })
 
-  // if we're in run mode with component
-  // testing then just pass this through
-  // without waiting on electron to be ready
-  return require('./run').ready(options)
+  return require('./run').run(options)
 }
 
 module.exports = {

packages/server/test/e2e/7_run_ct_spec.js

@@ -0,0 +1,12 @@
+const e2e = require('../support/helpers/e2e').default
+
+describe('run-ct', () => {
+  e2e.setup()
+
+  e2e.it('reports correct exit code when failing', {
+    spec: 'simple_failing_spec.js',
+    testingType: 'component',
+    snapshot: false,
+    expectedExitCode: 2,
+  })
+})

packages/server/test/support/fixtures/projects/e2e/cypress/component/simple_failing_spec.js

@@ -0,0 +1,10 @@
+/* eslint-disable no-undef */
+describe('simple failing spec', () => {
+  it('fails1', () => {
+    cy.wrap(true, { timeout: 100 }).should('be.false')
+  })
+
+  it('fails2', () => {
+    throw new Error('fails2')
+  })
+})

packages/server/test/support/fixtures/projects/e2e/cypress/plugins/index.js

@@ -6,12 +6,23 @@ const path = require('path')
 const Promise = require('bluebird')
 const { useFixedBrowserLaunchSize } = require('../../../utils')
 
+const { startDevServer } = require('@cypress/webpack-dev-server')
+
+const webpackConfig = {
+  output: {
+    publicPath: '/',
+  },
+  devServer: {
+    publicPath: '/',
+  },
+}
+
 /**
  * @type {Cypress.PluginConfig}
  */
 module.exports = (on, config) => {
-  if (config.testingType !== 'e2e') {
-    throw Error(`This is an e2e testing project. testingType should be 'e2e'. Received ${config.testingType}`)
+  if (config.testingType === 'component') {
+    on('dev-server:start', (options) => startDevServer({ options, webpackConfig }))
   }
 
   let performance = {

packages/server/test/support/helpers/e2e.ts

@@ -534,8 +534,9 @@ const e2e = {
           return spec
         }
 
-        // TODO would not work for component tests
-        return path.join(options.project, 'cypress', 'integration', spec)
+        const specDir = options.testingType === 'component' ? 'component' : 'integration'
+
+        return path.join(options.project, 'cypress', specDir, spec)
       })
 
       // normalize the path to the spec
@@ -552,7 +553,7 @@ const e2e = {
       // hides a user warning to go through NPM module
       `--cwd=${process.cwd()}`,
       `--run-project=${options.project}`,
-      `--testingType=e2e`,
+      `--testingType=${options.testingType || 'e2e'}`,
     ]
 
     if (options.testingType === 'component') {