diff --git a/circle.yml b/circle.yml index 5e4249a49b4d..74e25e1cdc4e 100644 --- a/circle.yml +++ b/circle.yml @@ -178,6 +178,14 @@ commands: mv ~/cypress/system-tests/node_modules /tmp/node_modules_cache/system-tests_node_modules mv ~/cypress/globbed_node_modules /tmp/node_modules_cache/globbed_node_modules + install-webkit-deps: + steps: + - run: + name: Install WebKit dependencies + command: | + npx playwright install webkit + npx playwright install-deps webkit + build-and-persist: description: Save entire folder as artifact for other jobs to run without reinstalling steps: @@ -462,6 +470,11 @@ commands: - install-chrome: channel: <> version: $(node ./scripts/get-browser-version.js chrome:<>) + - when: + condition: + equal: [ webkit, << parameters.browser >> ] + steps: + - install-webkit-deps - run: name: Run driver tests in Cypress environment: @@ -470,11 +483,6 @@ commands: echo Current working directory is $PWD echo Total containers $CIRCLE_NODE_TOTAL - if [[ "<>" = "webkit" ]]; then - npx playwright install webkit - npx playwright install-deps webkit - fi - if [[ -v MAIN_RECORD_KEY ]]; then # internal PR if <>; then @@ -610,6 +618,11 @@ commands: steps: - restore_cached_workspace - restore_cached_system_tests_deps + - when: + condition: + equal: [ webkit, << parameters.browser >> ] + steps: + - install-webkit-deps - run: name: Run system tests command: | @@ -1448,6 +1461,13 @@ jobs: - run-system-tests: browser: firefox + system-tests-webkit: + <<: *defaults + parallelism: 8 + steps: + - run-system-tests: + browser: webkit + system-tests-non-root: <<: *defaults steps: @@ -2363,6 +2383,10 @@ linux-x64-workflow: &linux-x64-workflow context: test-runner:performance-tracking requires: - system-tests-node-modules-install + - system-tests-webkit: + context: test-runner:performance-tracking + requires: + - system-tests-node-modules-install - system-tests-non-root: context: test-runner:performance-tracking executor: non-root-docker-user diff --git a/packages/server/lib/browsers/chrome.ts b/packages/server/lib/browsers/chrome.ts index 2a70a0b94105..6fcb3c6e47ce 100644 --- a/packages/server/lib/browsers/chrome.ts +++ b/packages/server/lib/browsers/chrome.ts @@ -15,12 +15,11 @@ import { fs } from '../util/fs' import { CdpAutomation, screencastOpts } from './cdp_automation' import * as protocol from './protocol' import utils from './utils' -import type { Browser } from './types' +import type { Browser, BrowserInstance } from './types' import { BrowserCriClient } from './browser-cri-client' -import type { LaunchedBrowser } from '@packages/launcher/lib/browsers' import type { CriClient } from './cri-client' import type { Automation } from '../automation' -import type { BrowserLaunchOpts, BrowserNewTabOpts, WriteVideoFrame } from '@packages/types' +import type { BrowserLaunchOpts, BrowserNewTabOpts, RunModeVideoApi } from '@packages/types' const debug = debugModule('cypress:server:browsers:chrome') @@ -249,10 +248,12 @@ const _disableRestorePagesPrompt = function (userDir) { .catch(() => { }) } -async function _recordVideo (cdpAutomation: CdpAutomation, writeVideoFrame: WriteVideoFrame, browserMajorVersion: number) { - const opts = browserMajorVersion >= CHROME_VERSION_WITH_FPS_INCREASE ? screencastOpts() : screencastOpts(1) +async function _recordVideo (cdpAutomation: CdpAutomation, videoOptions: RunModeVideoApi, browserMajorVersion: number) { + const screencastOptions = browserMajorVersion >= CHROME_VERSION_WITH_FPS_INCREASE ? screencastOpts() : screencastOpts(1) - await cdpAutomation.startVideoRecording(writeVideoFrame, opts) + const { writeVideoFrame } = await videoOptions.useFfmpegVideoController() + + await cdpAutomation.startVideoRecording(writeVideoFrame, screencastOptions) } // a utility function that navigates to the given URL @@ -552,7 +553,7 @@ export = { if (!options.url) throw new Error('Missing url in connectToNewSpec') - await this.attachListeners(browser, options.url, pageCriClient, automation, options) + await this.attachListeners(options.url, pageCriClient, automation, options) }, async connectToExisting (browser: Browser, options: BrowserLaunchOpts, automation: Automation) { @@ -570,9 +571,13 @@ export = { await this._setAutomation(pageCriClient, automation, browserCriClient.resetBrowserTargets, options) }, - async attachListeners (browser: Browser, url: string, pageCriClient: CriClient, automation: Automation, options: BrowserLaunchOpts | BrowserNewTabOpts) { + async attachListeners (url: string, pageCriClient: CriClient, automation: Automation, options: BrowserLaunchOpts | BrowserNewTabOpts) { + const browserCriClient = this._getBrowserCriClient() + if (!browserCriClient) throw new Error('Missing browserCriClient in attachListeners') + debug('attaching listeners to chrome %o', { url, options }) + const cdpAutomation = await this._setAutomation(pageCriClient, automation, browserCriClient.resetBrowserTargets, options) await pageCriClient.send('Page.enable') @@ -580,7 +585,7 @@ export = { await options['onInitializeNewBrowserTab']?.() await Promise.all([ - options.writeVideoFrame && this._recordVideo(cdpAutomation, options.writeVideoFrame, browser.majorVersion), + options.videoApi && this._recordVideo(cdpAutomation, options.videoApi, Number(options.browser.majorVersion)), this._handleDownloads(pageCriClient, options.downloadsFolder, automation), ]) @@ -590,9 +595,11 @@ export = { await this._handlePausedRequests(pageCriClient) _listenForFrameTreeChanges(pageCriClient) } + + return cdpAutomation }, - async open (browser: Browser, url, options: BrowserLaunchOpts, automation: Automation): Promise { + async open (browser: Browser, url, options: BrowserLaunchOpts, automation: Automation): Promise { const { isTextTerminal } = options const userDir = utils.getProfileDir(browser, isTextTerminal) @@ -644,7 +651,7 @@ export = { // first allows us to connect the remote interface, // start video recording and then // we will load the actual page - const launchedBrowser = await launch(browser, 'about:blank', port, args) as LaunchedBrowser & { browserCriClient: BrowserCriClient} + const launchedBrowser = await launch(browser, 'about:blank', port, args) as unknown as BrowserInstance & { browserCriClient: BrowserCriClient} la(launchedBrowser, 'did not get launched browser instance') @@ -671,7 +678,6 @@ export = { launchedBrowser.browserCriClient = browserCriClient - /* @ts-expect-error */ launchedBrowser.kill = (...args) => { debug('closing remote interface client') @@ -686,7 +692,7 @@ export = { const pageCriClient = await browserCriClient.attachToTargetUrl('about:blank') - await this.attachListeners(browser, url, pageCriClient, automation, options) + await this.attachListeners(url, pageCriClient, automation, options) // return the launched browser process // with additional method to close the remote connection diff --git a/packages/server/lib/browsers/electron.ts b/packages/server/lib/browsers/electron.ts index 7f3eba817379..421e58297d0b 100644 --- a/packages/server/lib/browsers/electron.ts +++ b/packages/server/lib/browsers/electron.ts @@ -11,7 +11,7 @@ import * as errors from '../errors' import type { Browser, BrowserInstance } from './types' import type { BrowserWindow, WebContents } from 'electron' import type { Automation } from '../automation' -import type { BrowserLaunchOpts, Preferences } from '@packages/types' +import type { BrowserLaunchOpts, Preferences, RunModeVideoApi } from '@packages/types' // TODO: unmix these two types type ElectronOpts = Windows.WindowOptions & BrowserLaunchOpts @@ -44,7 +44,7 @@ const tryToCall = function (win, method) { } } -const _getAutomation = async function (win, options, parent) { +const _getAutomation = async function (win, options: BrowserLaunchOpts, parent) { async function sendCommand (method: CdpCommand, data?: object) { return tryToCall(win, () => { return win.webContents.debugger.sendCommand @@ -72,7 +72,7 @@ const _getAutomation = async function (win, options, parent) { // after upgrading to Electron 8, CDP screenshots can hang if a screencast is not also running // workaround: start and stop screencasts between screenshots // @see https://github.com/cypress-io/cypress/pull/6555#issuecomment-596747134 - if (!options.writeVideoFrame) { + if (!options.videoApi) { await sendCommand('Page.startScreencast', screencastOpts()) const ret = await fn(message, data) @@ -109,6 +109,12 @@ function _installExtensions (win: BrowserWindow, extensionPaths: string[], optio })) } +async function recordVideo (cdpAutomation: CdpAutomation, videoApi: RunModeVideoApi) { + const { writeVideoFrame } = await videoApi.useFfmpegVideoController() + + await cdpAutomation.startVideoRecording(writeVideoFrame) +} + export = { _defaultOptions (projectRoot: string | undefined, state: Preferences, options: BrowserLaunchOpts, automation: Automation): ElectronOpts { const _this = this @@ -179,7 +185,7 @@ export = { _getAutomation, - async _render (url: string, automation: Automation, preferences, options: { projectRoot?: string, isTextTerminal: boolean }) { + async _render (url: string, automation: Automation, preferences, options: ElectronOpts) { const win = Windows.create(options.projectRoot, preferences) if (preferences.browser.isHeadless) { @@ -192,7 +198,7 @@ export = { win.maximize() } - const launched = await this._launch(win, url, automation, preferences) + const launched = await this._launch(win, url, automation, preferences, options.videoApi) automation.use(await _getAutomation(win, preferences, automation)) @@ -225,7 +231,7 @@ export = { return this._launch(win, url, automation, electronOptions) }, - async _launch (win: BrowserWindow, url: string, automation: Automation, options) { + async _launch (win: BrowserWindow, url: string, automation: Automation, options: ElectronOpts, videoApi?: RunModeVideoApi) { if (options.show) { menu.set({ withInternalDevTools: true }) } @@ -239,9 +245,7 @@ export = { this._attachDebugger(win.webContents) - let ua - - ua = options.userAgent + const ua = options.userAgent if (ua) { this._setUserAgent(win.webContents, ua) @@ -273,8 +277,9 @@ export = { const cdpAutomation = await this._getAutomation(win, options, automation) automation.use(cdpAutomation) + await Promise.all([ - options.writeVideoFrame && cdpAutomation.startVideoRecording(options.writeVideoFrame), + videoApi && recordVideo(cdpAutomation, videoApi), this._handleDownloads(win, options.downloadsFolder, automation), ]) @@ -445,7 +450,7 @@ export = { }) }, - async connectToNewSpec (browser, options, automation) { + async connectToNewSpec (browser: Browser, options: ElectronOpts, automation: Automation) { if (!options.url) throw new Error('Missing url in connectToNewSpec') await this.open(browser, options.url, options, automation) @@ -480,10 +485,7 @@ export = { debug('launching browser window to url: %s', url) - const win = await this._render(url, automation, preferences, { - projectRoot: options.projectRoot, - isTextTerminal: options.isTextTerminal, - }) + const win = await this._render(url, automation, preferences, electronOptions) await _installExtensions(win, launchOptions.extensions, electronOptions) diff --git a/packages/server/lib/browsers/firefox.ts b/packages/server/lib/browsers/firefox.ts index 4ebcd132f36e..d1fb393d7ea0 100644 --- a/packages/server/lib/browsers/firefox.ts +++ b/packages/server/lib/browsers/firefox.ts @@ -4,7 +4,7 @@ import Debug from 'debug' import getPort from 'get-port' import path from 'path' import urlUtil from 'url' -import { debug as launcherDebug, launch, LaunchedBrowser } from '@packages/launcher/lib/browsers' +import { debug as launcherDebug, launch } from '@packages/launcher/lib/browsers' import { doubleEscape } from '@packages/launcher/lib/windows' import FirefoxProfile from 'firefox-profile' import * as errors from '../errors' @@ -20,7 +20,7 @@ import type { BrowserCriClient } from './browser-cri-client' import type { Automation } from '../automation' import { getCtx } from '@packages/data-context' import { getError } from '@packages/errors' -import type { BrowserLaunchOpts, BrowserNewTabOpts } from '@packages/types' +import type { BrowserLaunchOpts, BrowserNewTabOpts, RunModeVideoApi } from '@packages/types' const debug = Debug('cypress:server:browsers:firefox') @@ -379,6 +379,12 @@ export function connectToExisting () { getCtx().onWarning(getError('UNEXPECTED_INTERNAL_ERROR', new Error('Attempting to connect to existing browser for Cypress in Cypress which is not yet implemented for firefox'))) } +async function recordVideo (videoApi: RunModeVideoApi) { + const { writeVideoFrame } = await videoApi.useFfmpegVideoController({ webmInput: true }) + + videoApi.onProjectCaptureVideoFrames(writeVideoFrame) +} + export async function open (browser: Browser, url: string, options: BrowserLaunchOpts, automation: Automation): Promise { // see revision comment here https://wiki.mozilla.org/index.php?title=WebDriver/RemoteProtocol&oldid=1234946 const hasCdp = browser.majorVersion >= 86 @@ -456,6 +462,7 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc utils.ensureCleanCache(browser, options.isTextTerminal), utils.writeExtension(browser, options.isTextTerminal, options.proxyUrl, options.socketIoRoute), utils.executeBeforeBrowserLaunch(browser, defaultLaunchOptions, options), + options.videoApi && recordVideo(options.videoApi), ]) if (Array.isArray(launchOptions.extensions)) { @@ -529,7 +536,7 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc // user can overwrite this default with these env vars or --height, --width arguments MOZ_HEADLESS_WIDTH: '1280', MOZ_HEADLESS_HEIGHT: '721', - }) as LaunchedBrowser & { browserCriClient: BrowserCriClient} + }) as unknown as BrowserInstance try { browserCriClient = await firefoxUtil.setup({ automation, extensions: launchOptions.extensions, url, foxdriverPort, marionettePort, remotePort, onError: options.onError, options }) @@ -543,7 +550,6 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc // monkey-patch the .kill method to that the CDP connection is closed const originalBrowserKill = browserInstance.kill - /* @ts-expect-error */ browserInstance.kill = (...args) => { debug('closing remote interface client') diff --git a/packages/server/lib/browsers/webkit-automation.ts b/packages/server/lib/browsers/webkit-automation.ts index cccf6b52caa7..be20e2bbb10c 100644 --- a/packages/server/lib/browsers/webkit-automation.ts +++ b/packages/server/lib/browsers/webkit-automation.ts @@ -3,6 +3,8 @@ import Debug from 'debug' import type playwright from 'playwright-webkit' import type { Automation } from '../automation' import { normalizeResourceType } from './cdp_automation' +import os from 'os' +import type { RunModeVideoApi } from '@packages/types' const debug = Debug('cypress:server:browsers:webkit-automation') @@ -90,26 +92,32 @@ export class WebKitAutomation { private constructor (public automation: Automation, private browser: playwright.Browser) {} // static initializer to avoid "not definitively declared" - static async create (automation: Automation, browser: playwright.Browser, initialUrl: string) { + static async create (automation: Automation, browser: playwright.Browser, initialUrl: string, videoApi?: RunModeVideoApi) { const wkAutomation = new WebKitAutomation(automation, browser) - await wkAutomation.reset(initialUrl) + await wkAutomation.reset(initialUrl, videoApi) return wkAutomation } - public async reset (newUrl?: string) { + public async reset (newUrl?: string, videoApi?: RunModeVideoApi) { debug('resetting playwright page + context %o', { newUrl }) // new context comes with new cache + storage const newContext = await this.browser.newContext({ ignoreHTTPSErrors: true, + recordVideo: videoApi && { + dir: os.tmpdir(), + size: { width: 1280, height: 720 }, + }, }) + const contextStarted = new Date const oldPwPage = this.page this.page = await newContext.newPage() this.context = this.page.context() this.attachListeners(this.page) + if (videoApi) this.recordVideo(videoApi, contextStarted) let promises: Promise[] = [] @@ -120,6 +128,40 @@ export class WebKitAutomation { if (promises.length) await Promise.all(promises) } + private recordVideo (videoApi: RunModeVideoApi, startedVideoCapture: Date) { + const _this = this + + videoApi.useVideoController({ + async endVideoCapture () { + const pwVideo = _this.page.video() + + if (!pwVideo) throw new Error('pw.page missing video in endVideoCapture, cannot save video') + + debug('ending video capture, closing page...') + + await Promise.all([ + // pwVideo.saveAs will not resolve until the page closes, presumably we do want to close it + _this.page.close(), + pwVideo.saveAs(videoApi.videoName), + ]) + }, + writeVideoFrame: () => { + throw new Error('writeVideoFrame called, but WebKit does not support streaming frame data.') + }, + async restart () { + throw new Error('Cannot restart WebKit video - WebKit cannot record video on multiple specs in single-tab mode.') + }, + postProcessFfmpegOptions: { + // WebKit seems to record at the highest possible frame rate, so filter out duplicate frames before compressing + // otherwise compressing with all these dupe frames can take a really long time + // https://stackoverflow.com/q/37088517/3474615 + outputOptions: ['-vsync vfr'], + videoFilters: 'mpdecimate', + }, + startedVideoCapture, + }) + } + private attachListeners (page: playwright.Page) { // emit preRequest to proxy page.on('request', (request) => { diff --git a/packages/server/lib/browsers/webkit.ts b/packages/server/lib/browsers/webkit.ts index 0eb3399a64b1..d40840d654c6 100644 --- a/packages/server/lib/browsers/webkit.ts +++ b/packages/server/lib/browsers/webkit.ts @@ -16,7 +16,7 @@ export async function connectToNewSpec (browser: Browser, options: BrowserNewTab automation.use(wkAutomation) wkAutomation.automation = automation await options.onInitializeNewBrowserTab() - await wkAutomation.reset(options.url) + await wkAutomation.reset(options.url, options.videoApi) } export function connectToExisting () { @@ -36,7 +36,7 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc headless: browser.isHeadless, }) - wkAutomation = await WebKitAutomation.create(automation, pwBrowser, url) + wkAutomation = await WebKitAutomation.create(automation, pwBrowser, url, options.videoApi) automation.use(wkAutomation) class WkInstance extends EventEmitter implements BrowserInstance { diff --git a/packages/server/lib/makeDataContext.ts b/packages/server/lib/makeDataContext.ts index eb2c4cb59688..093a4965b202 100644 --- a/packages/server/lib/makeDataContext.ts +++ b/packages/server/lib/makeDataContext.ts @@ -53,8 +53,8 @@ export function makeDataContext (options: MakeDataContextOptions): DataContext { async focusActiveBrowserWindow () { return openProject.sendFocusBrowserMessage() }, - relaunchBrowser () { - return openProject.relaunchBrowser ? openProject.relaunchBrowser() : null + async relaunchBrowser () { + await openProject.relaunchBrowser() }, }, appApi: { @@ -75,8 +75,8 @@ export function makeDataContext (options: MakeDataContextOptions): DataContext { }, }, projectApi: { - launchProject (browser: FoundBrowser, spec: Cypress.Spec, options: OpenProjectLaunchOpts) { - return openProject.launch({ ...browser }, spec, options) + async launchProject (browser: FoundBrowser, spec: Cypress.Spec, options: OpenProjectLaunchOpts) { + await openProject.launch({ ...browser }, spec, options) }, openProjectCreate (args: InitializeProjectOptions, options: OpenProjectLaunchOptions) { return openProject.create(args.projectRoot, args, options) diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index cb00eb5c3fc0..8d7843ecc1b3 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -21,7 +21,7 @@ import random from '../util/random' import system from '../util/system' import chromePolicyCheck from '../util/chrome_policy_check' import * as objUtils from '../util/obj_utils' -import type { SpecWithRelativeRoot, SpecFile, TestingType, OpenProjectLaunchOpts, FoundBrowser, WriteVideoFrame } from '@packages/types' +import type { SpecWithRelativeRoot, SpecFile, TestingType, OpenProjectLaunchOpts, FoundBrowser, BrowserVideoController, VideoRecording, ProcessOptions } from '@packages/types' import type { Cfg } from '../project-base' import type { Browser } from '../browsers/types' import * as printResults from '../util/print-run' @@ -40,7 +40,6 @@ let exitEarly = (err) => { earlyExitErr = err } let earlyExitErr: Error -let currentWriteVideoFrameCallback: WriteVideoFrame let currentSetScreenshotMetadata: SetScreenshotMetadata const debug = Debug('cypress:server:run') @@ -223,13 +222,24 @@ async function trashAssets (config: Cfg) { } } -async function createVideoRecording (videoName, options = {}) { +async function startVideoRecording (options: { previous?: VideoRecording, project: Project, spec: SpecWithRelativeRoot, videosFolder: string }): Promise { + if (!options.videosFolder) throw new Error('Missing videoFolder for recording') + + function videoPath (suffix: string) { + return path.join(options.videosFolder, options.spec.relativeToCommonRoot + suffix) + } + + const videoName = videoPath('.mp4') + const compressedVideoName = videoPath('-compressed.mp4') + const outputDir = path.dirname(videoName) const onError = _.once((err) => { // catch video recording failures and log them out // but don't let this affect the run at all - return errors.warning('VIDEO_RECORDING_FAILED', err) + errors.warning('VIDEO_RECORDING_FAILED', err) + + return undefined }) try { @@ -238,45 +248,65 @@ async function createVideoRecording (videoName, options = {}) { onError(err) } - return videoCapture.start(videoName, _.extend({}, options, { onError })) -} + if (options.previous) { + debug('in single-tab mode, re-using previous videoController') -const getVideoRecordingDelay = function (startedVideoCapture) { - if (startedVideoCapture) { - return DELAY_TO_LET_VIDEO_FINISH_MS - } + Object.assign(options.previous.api, { + videoName, + compressedVideoName, + onError, + }) - return 0 -} + await options.previous.controller?.restart().catch(onError) -async function maybeStartVideoRecording (options: { spec: SpecWithRelativeRoot, browser: Browser, video: boolean, videosFolder: string }) { - const { spec, browser, video, videosFolder } = options + return options.previous + } - debug(`video recording has been ${video ? 'enabled' : 'disabled'}. video: %s`, video) + let ffmpegController: BrowserVideoController + let _ffmpegOpts: Pick - if (!video) { - return - } + const videoRecording: VideoRecording = { + api: { + onError, + videoName, + compressedVideoName, + async useFfmpegVideoController (ffmpegOpts) { + _ffmpegOpts = ffmpegOpts || _ffmpegOpts + ffmpegController = await videoCapture.start({ ...videoRecording.api, ..._ffmpegOpts }) + + // This wrapper enables re-binding writeVideoFrame to a new video stream when running in single-tab mode. + const controllerWrap: BrowserVideoController = { + ...ffmpegController, + writeVideoFrame: function writeVideoFrameWrap (data) { + if (!ffmpegController) throw new Error('missing ffmpegController in writeVideoFrameWrap') + + ffmpegController.writeVideoFrame(data) + }, + async restart () { + await videoRecording.api.useFfmpegVideoController(_ffmpegOpts) + }, + } - if (!videosFolder) { - throw new Error('Missing videoFolder for recording') - } + videoRecording.api.useVideoController(controllerWrap) - const videoPath = (suffix) => { - return path.join(videosFolder, spec.relativeToCommonRoot + suffix) + return controllerWrap + }, + useVideoController (videoController) { + debug('setting videoController for videoRecording %o', videoRecording) + videoRecording.controller = videoController + }, + onProjectCaptureVideoFrames (fn) { + options.project.on('capture:video:frames', fn) + }, + }, + controller: undefined, } - const videoName = videoPath('.mp4') - const compressedVideoName = videoPath('-compressed.mp4') - const props = await createVideoRecording(videoName, { webmInput: browser.family === 'firefox' }) + options.project.videoRecording = videoRecording - return { - videoName, - compressedVideoName, - endVideoCapture: props.endVideoCapture, - writeVideoFrame: props.writeVideoFrame, - startedVideoCapture: props.startedVideoCapture, - } + debug('created videoRecording %o', { videoRecording }) + + return videoRecording } const warnVideoRecordingFailed = (err) => { @@ -285,50 +315,45 @@ const warnVideoRecordingFailed = (err) => { errors.warning('VIDEO_POST_PROCESSING_FAILED', err) } -function navigateToNextSpec (spec) { - debug('navigating to next spec %s', spec) - - return openProject.changeUrlToSpec(spec) -} - -async function postProcessRecording (name, cname, videoCompression, shouldUploadVideo, quiet, ffmpegChaptersConfig) { - debug('ending the video recording %o', { name, videoCompression, shouldUploadVideo }) +async function postProcessRecording (options: { quiet: boolean, videoCompression: number | boolean, shouldUploadVideo: boolean, processOptions: Omit }) { + debug('ending the video recording %o', options) // once this ended promises resolves // then begin processing the file // dont process anything if videoCompress is off // or we've been told not to upload the video - if (videoCompression === false || shouldUploadVideo === false) { + if (options.videoCompression === false || options.shouldUploadVideo === false) { return } + const processOptions: ProcessOptions = { + ...options.processOptions, + videoCompression: Number(options.videoCompression), + } + function continueProcessing (onProgress?: (progress: number) => void) { - return videoCapture.process(name, cname, videoCompression, ffmpegChaptersConfig, onProgress) + return videoCapture.process({ ...processOptions, onProgress }) } - if (quiet) { + if (options.quiet) { return continueProcessing() } - const { onProgress } = printResults.displayVideoProcessingProgress({ name, videoCompression }) + const { onProgress } = printResults.displayVideoProcessingProgress(processOptions) return continueProcessing(onProgress) } -function launchBrowser (options: { browser: Browser, spec: SpecWithRelativeRoot, writeVideoFrame?: WriteVideoFrame, setScreenshotMetadata: SetScreenshotMetadata, project: Project, screenshots: ScreenshotMetadata[], projectRoot: string, shouldLaunchNewTab: boolean, onError: (err: Error) => void }) { - const { browser, spec, setScreenshotMetadata, project, screenshots, projectRoot, shouldLaunchNewTab, onError } = options +function launchBrowser (options: { browser: Browser, spec: SpecWithRelativeRoot, setScreenshotMetadata: SetScreenshotMetadata, screenshots: ScreenshotMetadata[], projectRoot: string, shouldLaunchNewTab: boolean, onError: (err: Error) => void, videoRecording?: VideoRecording }) { + const { browser, spec, setScreenshotMetadata, screenshots, projectRoot, shouldLaunchNewTab, onError } = options const warnings = {} - if (options.writeVideoFrame && browser.family === 'firefox') { - project.on('capture:video:frames', options.writeVideoFrame) - } - const browserOpts: OpenProjectLaunchOpts = { projectRoot, shouldLaunchNewTab, onError, - writeVideoFrame: options.writeVideoFrame, + videoApi: options.videoRecording?.api, automationMiddleware: { onBeforeRequest (message, data) { if (message === 'take:screenshot') { @@ -417,34 +442,13 @@ function listenForProjectEnd (project, exit): Bluebird { }) } -/** - * In CT mode, browser do not relaunch. - * In browser laucnh is where we wire the new video - * recording callback. - * This has the effect of always hitting the first specs - * video callback. - * - * This allows us, if we need to, to call a different callback - * in the same browser - */ -function writeVideoFrameCallback (data: Buffer) { - return currentWriteVideoFrameCallback(data) -} - -function waitForBrowserToConnect (options: { project: Project, socketId: string, onError: (err: Error) => void, writeVideoFrame?: WriteVideoFrame, spec: SpecWithRelativeRoot, isFirstSpec: boolean, testingType: string, experimentalSingleTabRunMode: boolean, browser: Browser, screenshots: ScreenshotMetadata[], projectRoot: string, shouldLaunchNewTab: boolean, webSecurity: boolean }) { +async function waitForBrowserToConnect (options: { project: Project, socketId: string, onError: (err: Error) => void, spec: SpecWithRelativeRoot, isFirstSpec: boolean, testingType: string, experimentalSingleTabRunMode: boolean, browser: Browser, screenshots: ScreenshotMetadata[], projectRoot: string, shouldLaunchNewTab: boolean, webSecurity: boolean, videoRecording?: VideoRecording }) { if (globalThis.CY_TEST_MOCK?.waitForBrowserToConnect) return Promise.resolve() - const { project, socketId, onError, writeVideoFrame, spec } = options + const { project, socketId, onError, spec } = options const browserTimeout = Number(process.env.CYPRESS_INTERNAL_BROWSER_CONNECT_TIMEOUT || 60000) let attempts = 0 - // short circuit current browser callback so that we - // can rewire it without relaunching the browser - if (writeVideoFrame) { - currentWriteVideoFrameCallback = writeVideoFrame - options.writeVideoFrame = writeVideoFrameCallback - } - // without this the run mode is only setting new spec // path for next spec in launch browser. // we need it to run on every spec even in single browser mode @@ -462,15 +466,15 @@ function waitForBrowserToConnect (options: { project: Project, socketId: string, if (options.experimentalSingleTabRunMode && options.testingType === 'component' && !options.isFirstSpec) { // reset browser state to match default behavior when opening/closing a new tab - return openProject.resetBrowserState().then(() => { - // If we do not launch the browser, - // we tell it that we are ready - // to receive the next spec - return navigateToNextSpec(options.spec) - }) + await openProject.resetBrowserState() + + // since we aren't re-launching the browser, we have to navigate to the next spec instead + debug('navigating to next spec %s', spec) + + return openProject.changeUrlToSpec(spec) } - const wait = () => { + const wait = async () => { debug('waiting for socket to connect and browser to launch...') return Bluebird.all([ @@ -507,7 +511,7 @@ function waitForBrowserToConnect (options: { project: Project, socketId: string, return wait() } -function waitForSocketConnection (project, id) { +function waitForSocketConnection (project: Project, id: string) { if (globalThis.CY_TEST_MOCK?.waitForSocketConnection) return debug('waiting for socket connection... %o', { id }) @@ -533,123 +537,134 @@ function waitForSocketConnection (project, id) { }) } -function waitForTestsToFinishRunning (options: { project: Project, screenshots: ScreenshotMetadata[], startedVideoCapture?: any, endVideoCapture?: () => Promise, videoName?: string, compressedVideoName?: string, videoCompression: number | false, videoUploadOnPasses: boolean, exit: boolean, spec: SpecWithRelativeRoot, estimated: number, quiet: boolean, config: Cfg, shouldKeepTabOpen: boolean, testingType: TestingType}) { +async function waitForTestsToFinishRunning (options: { project: Project, screenshots: ScreenshotMetadata[], videoCompression: number | false, videoUploadOnPasses: boolean, exit: boolean, spec: SpecWithRelativeRoot, estimated: number, quiet: boolean, config: Cfg, shouldKeepTabOpen: boolean, testingType: TestingType, videoRecording?: VideoRecording }) { if (globalThis.CY_TEST_MOCK?.waitForTestsToFinishRunning) return Promise.resolve(globalThis.CY_TEST_MOCK.waitForTestsToFinishRunning) - const { project, screenshots, startedVideoCapture, endVideoCapture, videoName, compressedVideoName, videoCompression, videoUploadOnPasses, exit, spec, estimated, quiet, config, shouldKeepTabOpen, testingType } = options + const { project, screenshots, videoRecording, videoCompression, videoUploadOnPasses, exit, spec, estimated, quiet, config, shouldKeepTabOpen, testingType } = options + + const results = await listenForProjectEnd(project, exit) + + debug('received project end %o', results) // https://github.com/cypress-io/cypress/issues/2370 // delay 1 second if we're recording a video to give // the browser padding to render the final frames // to avoid chopping off the end of the video - const delay = getVideoRecordingDelay(startedVideoCapture) - - return listenForProjectEnd(project, exit) - .delay(delay) - .then(async (results) => { - _.defaults(results, { - error: null, - hooks: null, - tests: null, - video: null, - screenshots: null, - reporterStats: null, - }) + const videoController = videoRecording?.controller - // dashboard told us to skip this spec - const skippedSpec = results.skippedSpec + debug('received videoController %o', { videoController }) - if (startedVideoCapture) { - results.video = videoName - } + if (videoController) { + debug('delaying to extend video %o', { DELAY_TO_LET_VIDEO_FINISH_MS }) + await Bluebird.delay(DELAY_TO_LET_VIDEO_FINISH_MS) + } - if (screenshots) { - results.screenshots = screenshots - } + _.defaults(results, { + error: null, + hooks: null, + tests: null, + video: null, + screenshots: null, + reporterStats: null, + }) - results.spec = spec + // dashboard told us to skip this spec + const skippedSpec = results.skippedSpec - const { tests, stats } = results - const attempts = _.flatMap(tests, (test) => test.attempts) + if (screenshots) { + results.screenshots = screenshots + } - // if we have a video recording - if (startedVideoCapture && tests && tests.length) { - // always set the video timestamp on tests - Reporter.setVideoTimestamp(startedVideoCapture, attempts) - } + results.spec = spec - let videoCaptureFailed = false + const { tests, stats } = results + const attempts = _.flatMap(tests, (test) => test.attempts) - if (endVideoCapture) { - try { - await endVideoCapture() - } catch (err) { - videoCaptureFailed = true - warnVideoRecordingFailed(err) - } - } - - await runEvents.execute('after:spec', config, spec, results) + let videoCaptureFailed = false - const videoExists = videoName ? await fs.pathExists(videoName) : false + // if we have a video recording + if (videoController) { + results.video = videoRecording!.api.videoName - if (startedVideoCapture && !videoExists) { - // the video file no longer exists at the path where we expect it, - // likely because the user deleted it in the after:spec event - debug(`No video found after spec ran - skipping processing. Video path: ${videoName}`) + if (tests && tests.length) { + // always set the video timestamp on tests + Reporter.setVideoTimestamp(videoController.startedVideoCapture, attempts) + } - results.video = null + try { + await videoController.endVideoCapture() + debug('ended video capture') + } catch (err) { + videoCaptureFailed = true + warnVideoRecordingFailed(err) } + } - const hasFailingTests = _.get(stats, 'failures') > 0 - // we should upload the video if we upload on passes (by default) - // or if we have any failures and have started the video - const shouldUploadVideo = !skippedSpec && videoUploadOnPasses === true || Boolean((startedVideoCapture && hasFailingTests)) + await runEvents.execute('after:spec', config, spec, results) + debug('executed after:spec') - results.shouldUploadVideo = shouldUploadVideo + const videoName = videoRecording?.api.videoName + const videoExists = videoName && await fs.pathExists(videoName) - if (!quiet && !skippedSpec) { - printResults.displayResults(results, estimated) - } + if (!videoExists) { + // the video file no longer exists at the path where we expect it, + // possibly because the user deleted it in the after:spec event + debug(`No video found after spec ran - skipping processing. Video path: ${videoName}`) - const project = openProject.getProject() + results.video = null + } - if (!project) throw new Error('Missing project!') + const hasFailingTests = _.get(stats, 'failures') > 0 + // we should upload the video if we upload on passes (by default) + // or if we have any failures and have started the video + const shouldUploadVideo = !skippedSpec && videoUploadOnPasses === true || Boolean((/* startedVideoCapture */ videoExists && hasFailingTests)) - // @ts-expect-error experimentalSingleTabRunMode only exists on the CT-specific config type - const usingExperimentalSingleTabMode = testingType === 'component' && config.experimentalSingleTabRunMode + results.shouldUploadVideo = shouldUploadVideo - if (usingExperimentalSingleTabMode) { - await project.server.destroyAut() - } + if (!quiet && !skippedSpec) { + printResults.displayResults(results, estimated) + } - // we do not support experimentalSingleTabRunMode for e2e - if (!usingExperimentalSingleTabMode) { - debug('attempting to close the browser tab') + // @ts-expect-error experimentalSingleTabRunMode only exists on the CT-specific config type + const usingExperimentalSingleTabMode = testingType === 'component' && config.experimentalSingleTabRunMode - await openProject.resetBrowserTabsForNextTest(shouldKeepTabOpen) + if (usingExperimentalSingleTabMode) { + await project.server.destroyAut() + } - debug('resetting server state') + // we do not support experimentalSingleTabRunMode for e2e + if (!usingExperimentalSingleTabMode) { + debug('attempting to close the browser tab') - project.server.reset() - } + await openProject.resetBrowserTabsForNextTest(shouldKeepTabOpen) - if (videoExists && !skippedSpec && endVideoCapture && !videoCaptureFailed) { - const ffmpegChaptersConfig = videoCapture.generateFfmpegChaptersConfig(results.tests) + debug('resetting server state') - await postProcessRecording( - videoName, - compressedVideoName, - videoCompression, + project.server.reset() + } + + if (videoExists && !skippedSpec && !videoCaptureFailed) { + const chaptersConfig = videoCapture.generateFfmpegChaptersConfig(results.tests) + + try { + debug('post processing recording') + await postProcessRecording({ shouldUploadVideo, quiet, - ffmpegChaptersConfig, - ) - .catch(warnVideoRecordingFailed) + videoCompression, + processOptions: { + compressedVideoName: videoRecording.api.compressedVideoName, + videoName, + chaptersConfig, + ...(videoRecording.controller!.postProcessFfmpegOptions || {}), + }, + }) + } catch (err) { + warnVideoRecordingFailed(err) } + } - return results - }) + return results } function screenshotMetadata (data, resp) { @@ -816,12 +831,21 @@ async function runSpec (config, spec: SpecWithRelativeRoot, options: { project: const screenshots = [] - const videoRecordProps = await maybeStartVideoRecording({ - spec, - browser, - video: options.video, - videosFolder: options.videosFolder, - }) + async function getVideoRecording () { + if (!options.video) return undefined + + const opts = { project, spec, videosFolder: options.videosFolder } + + if (config.experimentalSingleTabRunMode && !isFirstSpec && project.videoRecording) { + // in single-tab mode, only the first spec needs to create a videoRecording object + // which is then re-used between specs + return await startVideoRecording({ ...opts, previous: project.videoRecording }) + } + + return await startVideoRecording(opts) + } + + const videoRecording = await getVideoRecording() // we know we're done running headlessly // when the renderer has connected and @@ -834,10 +858,7 @@ async function runSpec (config, spec: SpecWithRelativeRoot, options: { project: project, estimated, screenshots, - videoName: videoRecordProps?.videoName, - compressedVideoName: videoRecordProps?.compressedVideoName, - endVideoCapture: videoRecordProps?.endVideoCapture, - startedVideoCapture: videoRecordProps?.startedVideoCapture, + videoRecording, exit: options.exit, testingType: options.testingType, videoCompression: options.videoCompression, @@ -852,7 +873,7 @@ async function runSpec (config, spec: SpecWithRelativeRoot, options: { project: browser, screenshots, onError, - writeVideoFrame: videoRecordProps?.writeVideoFrame, + videoRecording, socketId: options.socketId, webSecurity: options.webSecurity, projectRoot: options.projectRoot, diff --git a/packages/server/lib/open_project.ts b/packages/server/lib/open_project.ts index c1f4175e425f..e3f43e2cb201 100644 --- a/packages/server/lib/open_project.ts +++ b/packages/server/lib/open_project.ts @@ -15,12 +15,15 @@ import { getSpecUrl } from './project_utils' import type { BrowserLaunchOpts, OpenProjectLaunchOptions, InitializeProjectOptions, OpenProjectLaunchOpts, FoundBrowser } from '@packages/types' import { DataContext, getCtx } from '@packages/data-context' import { autoBindDebug } from '@packages/data-context/src/util' +import type { BrowserInstance } from './browsers/types' const debug = Debug('cypress:server:open_project') export class OpenProject { private projectBase: ProjectBase | null = null - relaunchBrowser: (() => Promise) | null = null + relaunchBrowser: (() => Promise) = () => { + throw new Error('bad relaunch') + } constructor () { return autoBindDebug(this) @@ -29,7 +32,9 @@ export class OpenProject { resetOpenProject () { this.projectBase?.__reset() this.projectBase = null - this.relaunchBrowser = null + this.relaunchBrowser = () => { + throw new Error('bad relaunch after reset') + } } reset () { diff --git a/packages/server/lib/project-base.ts b/packages/server/lib/project-base.ts index bdc4e0bb55c6..d7b7ede00459 100644 --- a/packages/server/lib/project-base.ts +++ b/packages/server/lib/project-base.ts @@ -19,7 +19,7 @@ import { SocketE2E } from './socket-e2e' import { ensureProp } from './util/class-helpers' import system from './util/system' -import type { BannersState, FoundBrowser, FoundSpec, OpenProjectLaunchOptions, ReceivedCypressOptions, ResolvedConfigurationOptions, TestingType } from '@packages/types' +import type { BannersState, FoundBrowser, FoundSpec, OpenProjectLaunchOptions, ReceivedCypressOptions, ResolvedConfigurationOptions, TestingType, VideoRecording } from '@packages/types' import { DataContext, getCtx } from '@packages/data-context' import { createHmac } from 'crypto' @@ -60,6 +60,7 @@ export class ProjectBase extends EE { private _recordTests?: any = null private _isServerOpen: boolean = false + public videoRecording?: VideoRecording public browser: any public options: OpenProjectLaunchOptions public testingType: Cypress.TestingType diff --git a/packages/server/lib/util/print-run.ts b/packages/server/lib/util/print-run.ts index a3032156dc6d..36db19dea534 100644 --- a/packages/server/lib/util/print-run.ts +++ b/packages/server/lib/util/print-run.ts @@ -452,7 +452,7 @@ function displayScreenshots (screenshots: Screenshot[] = []) { console.log('') } -export function displayVideoProcessingProgress (opts: { name: string, videoCompression: number | false }) { +export function displayVideoProcessingProgress (opts: { videoName: string, videoCompression: number | false }) { console.log('') terminal.header('Video', { @@ -508,7 +508,7 @@ export function displayVideoProcessingProgress (opts: { name: string, videoCompr table.push([ gray('-'), gray('Finished processing:'), - `${formatPath(opts.name, getWidth(table, 2), 'cyan')}`, + `${formatPath(opts.videoName, getWidth(table, 2), 'cyan')}`, gray(dur), ]) diff --git a/packages/server/lib/util/process_profiler.ts b/packages/server/lib/util/process_profiler.ts index d228c7b30bd9..a2a01bb49084 100644 --- a/packages/server/lib/util/process_profiler.ts +++ b/packages/server/lib/util/process_profiler.ts @@ -51,7 +51,7 @@ export const _groupCyProcesses = ({ list }: si.Systeminformation.ProcessesData) const isBrowserProcess = (proc: Process): boolean => { const instance = browsers.getBrowserInstance() // electron will return a list of pids, since it's not a hierarchy - const pids: number[] = instance.allPids ? instance.allPids : [instance.pid] + const pids: number[] = instance?.allPids ? instance.allPids : [instance?.pid] return (pids.includes(proc.pid)) || isParentProcessInGroup(proc, 'browser') diff --git a/packages/server/lib/video_capture.ts b/packages/server/lib/video_capture.ts index 0bdad7cfbe0a..c673da2d237e 100644 --- a/packages/server/lib/video_capture.ts +++ b/packages/server/lib/video_capture.ts @@ -7,7 +7,7 @@ import Bluebird from 'bluebird' import { path as ffmpegPath } from '@ffmpeg-installer/ffmpeg' import BlackHoleStream from 'black-hole-stream' import { fs } from './util/fs' -import type { WriteVideoFrame } from '@packages/types' +import type { ProcessOptions, WriteVideoFrame } from '@packages/types' const debug = Debug('cypress:server:video') const debugVerbose = Debug('cypress-verbose:server:video') @@ -21,7 +21,7 @@ ffmpeg.setFfmpegPath(ffmpegPath) const deferredPromise = function () { let reject let resolve - const promise = new Bluebird((_resolve, _reject) => { + const promise = new Promise((_resolve, _reject) => { resolve = _resolve reject = _reject }) @@ -31,7 +31,7 @@ const deferredPromise = function () { export function generateFfmpegChaptersConfig (tests) { if (!tests) { - return null + return } const configString = tests.map((test) => { @@ -108,14 +108,16 @@ export function copy (src, dest) { }) } -type StartOptions = { +export type StartOptions = { + // Path to write video to. + videoName: string // If set, expect input frames as webm chunks. webmInput?: boolean // Callback for asynchronous errors in video processing/compression. onError?: (err: Error, stdout: string, stderr: string) => void } -export function start (name, options: StartOptions = {}) { +export function start (options: StartOptions) { const pt = new stream.PassThrough() const ended = deferredPromise() let done = false @@ -188,8 +190,10 @@ export function start (name, options: StartOptions = {}) { debugFrames('writing video frame') if (wantsWrite) { - if (!(wantsWrite = pt.write(data))) { - return pt.once('drain', () => { + wantsWrite = pt.write(data) + if (!wantsWrite) { + // ffmpeg stream isn't accepting data, so drop frames until the stream is ready to accept data + pt.once('drain', () => { debugFrames('video stream drained') wantsWrite = true @@ -203,7 +207,7 @@ export function start (name, options: StartOptions = {}) { } const startCapturing = () => { - return new Bluebird((resolve) => { + return new Promise((resolve) => { const cmd = ffmpeg({ source: pt, priority: 20, @@ -256,7 +260,7 @@ export function start (name, options: StartOptions = {}) { .inputOptions('-use_wallclock_as_timestamps 1') } - return cmd.save(name) + return cmd.save(options.videoName) }) } @@ -268,22 +272,21 @@ export function start (name, options: StartOptions = {}) { endVideoCapture, writeVideoFrame, startedVideoCapture, + restart: () => { + throw new Error('restart cannot be called on a plain ffmpeg stream') + }, } }) } -// Progress callback called with percentage `0 <= p <= 1` of compression progress. -type OnProgress = (p: number) => void - -export async function process (name, cname, videoCompression, ffmpegchaptersConfig, onProgress: OnProgress = function () {}) { +export async function process (options: ProcessOptions) { let total = null - const metaFileName = `${name}.meta` - const addChaptersMeta = ffmpegchaptersConfig && await fs.writeFile(metaFileName, ffmpegchaptersConfig).then(() => true) + const metaFileName = `${options.videoName}.meta` + const addChaptersMeta = options.chaptersConfig && await fs.writeFile(metaFileName, options.chaptersConfig).then(() => true) - return new Bluebird((resolve, reject) => { - debug('processing video from %s to %s video compression %o', - name, cname, videoCompression) + return new Promise((resolve, reject) => { + debug('processing video %o', options) const command = ffmpeg({ priority: 20, @@ -319,12 +322,13 @@ export async function process (name, cname, videoCompression, ffmpegchaptersConf '-preset fast', // Compression Rate Factor is essentially the quality dial; 0 would be lossless // (big files), while 51 (the maximum) would lead to low quality (and small files). - `-crf ${videoCompression}`, + `-crf ${options.videoCompression}`, // Discussion of pixel formats is beyond the scope of these comments. See // https://en.wikipedia.org/wiki/Chroma_subsampling if you want the gritty details. // Short version: yuv420p is a standard video format supported everywhere. '-pix_fmt yuv420p', + ...(options.outputOptions || []), ] if (addChaptersMeta) { @@ -332,10 +336,13 @@ export async function process (name, cname, videoCompression, ffmpegchaptersConf outputOptions.push('-map_metadata 1') } - command.input(name) + let chain = command.input(options.videoName) .videoCodec('libx264') .outputOptions(outputOptions) - // .videoFilters("crop='floor(in_w/2)*2:floor(in_h/2)*2'") + + if (options.videoFilters) chain = chain.videoFilters(options.videoFilters) + + chain .on('start', (command) => { debug('compression started %o', { command }) }) @@ -361,7 +368,7 @@ export async function process (name, cname, videoCompression, ffmpegchaptersConf const percent = progressed / total if (percent < 1) { - return onProgress(percent) + return options.onProgress?.(percent) } }) .on('error', (err, stdout, stderr) => { @@ -373,10 +380,10 @@ export async function process (name, cname, videoCompression, ffmpegchaptersConf debug('compression ended') // we are done progressing - onProgress(1) + options.onProgress?.(1) // rename and obliterate the original - await fs.move(cname, name, { + await fs.move(options.compressedVideoName, options.videoName, { overwrite: true, }) @@ -385,6 +392,6 @@ export async function process (name, cname, videoCompression, ffmpegchaptersConf } resolve() - }).save(cname) + }).save(options.compressedVideoName) }) } diff --git a/packages/server/test/integration/cypress_spec.js b/packages/server/test/integration/cypress_spec.js index 55b957073da4..a69d3aa3815b 100644 --- a/packages/server/test/integration/cypress_spec.js +++ b/packages/server/test/integration/cypress_spec.js @@ -975,6 +975,7 @@ describe('lib/cypress', () => { // and only then navigates to that URL sinon.stub(chromeBrowser, '_navigateUsingCRI').resolves() sinon.stub(chromeBrowser, '_handleDownloads').resolves() + sinon.stub(chromeBrowser, '_recordVideo').resolves() sinon.stub(chromeBrowser, '_setAutomation').returns() @@ -1004,6 +1005,7 @@ describe('lib/cypress', () => { expect(chromeBrowser._navigateUsingCRI).to.have.been.calledOnce expect(chromeBrowser._setAutomation).to.have.been.calledOnce + expect(chromeBrowser._recordVideo).to.have.been.calledOnce expect(BrowserCriClient.create).to.have.been.calledOnce expect(browserCriClient.attachToTargetUrl).to.have.been.calledOnce @@ -1011,9 +1013,7 @@ describe('lib/cypress', () => { }) it('electron', function () { - const writeVideoFrame = sinon.stub() - - videoCapture.start.returns({ writeVideoFrame }) + videoCapture.start.returns() return cypress.start([ `--run-project=${this.pluginBrowser}`, @@ -1024,7 +1024,6 @@ describe('lib/cypress', () => { browser: 'electron', foo: 'bar', onNewWindow: sinon.match.func, - writeVideoFrame: sinon.match.func, }) this.expectExitWith(0) diff --git a/packages/server/test/integration/video_capture_spec.ts b/packages/server/test/integration/video_capture_spec.ts index 74c180ecdea5..451879aa09c2 100644 --- a/packages/server/test/integration/video_capture_spec.ts +++ b/packages/server/test/integration/video_capture_spec.ts @@ -4,10 +4,10 @@ import path from 'path' import fse from 'fs-extra' import os from 'os' -async function startSpiedVideoCapture (filename, options = {}) { - const props = await videoCapture.start(filename, options) +async function startSpiedVideoCapture (videoName, options = {}) { + const props = await videoCapture.start({ videoName, ...options }) - const END_OF_FILE_ERROR = `ffmpeg exited with code 1: Output #0, mp4, to '${filename}': + const END_OF_FILE_ERROR = `ffmpeg exited with code 1: Output #0, mp4, to '${videoName}': Output file #0 does not contain any stream\n` sinon.spy(props._pt, 'write') @@ -57,6 +57,7 @@ describe('Video Capture', () => { writeVideoFrameAsBuffer(''), ] + // @ts-ignore expect(_pt.write.lastCall).calledWith(buf2) await expect(endVideoCapture()).rejectedWith(END_OF_FILE_ERROR) diff --git a/packages/server/test/performance/cy_visit_performance_spec.js b/packages/server/test/performance/cy_visit_performance_spec.js index a94e794a5d3c..35cbd75f859d 100644 --- a/packages/server/test/performance/cy_visit_performance_spec.js +++ b/packages/server/test/performance/cy_visit_performance_spec.js @@ -24,6 +24,7 @@ context('cy.visit performance tests', function () { } systemTests.it('passes', { + browser: '!webkit', // TODO(webkit): does this really need to run in all browsers? currently it's broken in webkit because we are missing deps configFile: 'cypress-performance.config.js', onStdout, spec: 'fast_visit.cy.js', diff --git a/packages/server/test/unit/browsers/cdp_automation_spec.ts b/packages/server/test/unit/browsers/cdp_automation_spec.ts index 608726185f68..5e7c380fff2a 100644 --- a/packages/server/test/unit/browsers/cdp_automation_spec.ts +++ b/packages/server/test/unit/browsers/cdp_automation_spec.ts @@ -67,6 +67,8 @@ context('lib/browsers/cdp_automation', () => { }) context('.CdpAutomation', () => { + let cdpAutomation: CdpAutomation + beforeEach(async function () { this.sendDebuggerCommand = sinon.stub() this.onFn = sinon.stub() @@ -76,14 +78,35 @@ context('lib/browsers/cdp_automation', () => { onRequestEvent: sinon.stub(), } - this.cdpAutomation = await CdpAutomation.create(this.sendDebuggerCommand, this.onFn, this.sendCloseTargetCommand, this.automation, false) + cdpAutomation = await CdpAutomation.create(this.sendDebuggerCommand, this.onFn, this.sendCloseTargetCommand, this.automation, false) this.sendDebuggerCommand .throws(new Error('not stubbed')) .withArgs('Browser.getVersion') .resolves() - this.onRequest = this.cdpAutomation.onRequest + this.onRequest = cdpAutomation.onRequest + }) + + describe('.startVideoRecording', function () { + // https://github.com/cypress-io/cypress/issues/9265 + it('respond ACK after receiving new screenshot frame', async function () { + const writeVideoFrame = sinon.stub() + const frameMeta = { data: Buffer.from('foo'), sessionId: '1' } + + this.onFn.withArgs('Page.screencastFrame').callsFake((e, fn) => { + fn(frameMeta) + }) + + const startScreencast = this.sendDebuggerCommand.withArgs('Page.startScreencast').resolves() + const screencastFrameAck = this.sendDebuggerCommand.withArgs('Page.screencastFrameAck').resolves() + + await cdpAutomation.startVideoRecording(writeVideoFrame) + + expect(startScreencast).to.have.been.calledWith('Page.startScreencast') + expect(writeVideoFrame).to.have.been.calledWithMatch((arg) => Buffer.isBuffer(arg) && arg.length > 0) + expect(screencastFrameAck).to.have.been.calledWith('Page.screencastFrameAck', { sessionId: frameMeta.sessionId }) + }) }) describe('.onNetworkRequestWillBeSent', function () { diff --git a/packages/server/test/unit/browsers/chrome_spec.js b/packages/server/test/unit/browsers/chrome_spec.js index 7f3faafb3259..428b0a6b6a20 100644 --- a/packages/server/test/unit/browsers/chrome_spec.js +++ b/packages/server/test/unit/browsers/chrome_spec.js @@ -323,20 +323,6 @@ describe('lib/browsers/chrome', () => { return expect(chrome.open({ isHeadless: true }, 'http://', openOpts, this.automation)).to.be.rejectedWith('Cypress requires at least Chrome 64.') }) - // https://github.com/cypress-io/cypress/issues/9265 - it('respond ACK after receiving new screenshot frame', function () { - const frameMeta = { data: Buffer.from('foo'), sessionId: '1' } - const write = sinon.stub() - const options = { writeVideoFrame: write } - - return this.onCriEvent('Page.screencastFrame', frameMeta, options) - .then(() => { - expect(this.pageCriClient.send).to.have.been.calledWith('Page.startScreencast') - expect(write).to.have.been.calledWithMatch((arg) => Buffer.isBuffer(arg) && arg.length > 0) - expect(this.pageCriClient.send).to.have.been.calledWith('Page.screencastFrameAck', { sessionId: frameMeta.sessionId }) - }) - }) - describe('downloads', function () { it('pushes create:download after download begins', function () { const downloadData = { @@ -520,7 +506,8 @@ describe('lib/browsers/chrome', () => { ...openOpts, url: 'https://www.google.com', downloadsFolder: '/tmp/folder', - writeVideoFrame: () => {}, + browser: {}, + videoApi: {}, onInitializeNewBrowserTab: () => { onInitializeNewBrowserTabCalled = true }, diff --git a/packages/server/test/unit/browsers/electron_spec.js b/packages/server/test/unit/browsers/electron_spec.js index 194bd1fecd3e..beab0a2cfe4f 100644 --- a/packages/server/test/unit/browsers/electron_spec.js +++ b/packages/server/test/unit/browsers/electron_spec.js @@ -19,6 +19,7 @@ describe('lib/browsers/electron', () => { this.url = 'https://foo.com' this.state = {} this.options = { + isTextTerminal: false, some: 'var', projectRoot: '/foo/', onWarning: sinon.stub().returns(), @@ -99,10 +100,10 @@ describe('lib/browsers/electron', () => { expect(_.keys(options)).to.deep.eq(preferencesKeys) - expect(electron._render.firstCall.args[3]).to.deep.eql({ - projectRoot: this.options.projectRoot, - isTextTerminal: this.options.isTextTerminal, - }) + const electronOptionsArg = electron._render.firstCall.args[3] + + expect(electronOptionsArg.projectRoot).to.eq(this.options.projectRoot) + expect(electronOptionsArg.isTextTerminal).to.eq(this.options.isTextTerminal) expect(electron._render).to.be.calledWith( this.url, diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 9be56583ac3b..e00a3d329c82 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -37,3 +37,5 @@ export * from './warning' export * from './modeOptions' export * from './git' + +export * from './video' diff --git a/packages/types/src/server.ts b/packages/types/src/server.ts index ce1540bb09b7..37186e4194f6 100644 --- a/packages/types/src/server.ts +++ b/packages/types/src/server.ts @@ -1,14 +1,13 @@ import type { FoundBrowser } from './browser' import type { ReceivedCypressOptions } from './config' import type { PlatformName } from './platform' - -export type WriteVideoFrame = (data: Buffer) => void +import type { RunModeVideoApi } from './video' export type OpenProjectLaunchOpts = { projectRoot: string shouldLaunchNewTab: boolean automationMiddleware: AutomationMiddleware - writeVideoFrame?: WriteVideoFrame + videoApi?: RunModeVideoApi onWarning: (err: Error) => void onError: (err: Error) => void } diff --git a/packages/types/src/video.ts b/packages/types/src/video.ts new file mode 100644 index 000000000000..7bc3fcf63688 --- /dev/null +++ b/packages/types/src/video.ts @@ -0,0 +1,60 @@ +// Progress callback called with percentage `0 <= p <= 1` of compression progress. +type OnProgress = (p: number) => void + +export type ProcessOptions = { + videoName: string + compressedVideoName: string + videoCompression: number + chaptersConfig?: string + onProgress?: OnProgress + outputOptions?: string[] + videoFilters?: string +} + +export type WriteVideoFrame = (data: Buffer) => void + +export type VideoRecording = { + api: RunModeVideoApi + controller?: BrowserVideoController +} + +/** + * Interface yielded by the browser to control video recording. + */ +export type BrowserVideoController = { + /** + * A function that resolves once the video is fully captured and flushed to disk. + */ + endVideoCapture: () => Promise + /** + * Timestamp of when the video capture started - used for chapter timestamps. + */ + startedVideoCapture: Date + postProcessFfmpegOptions?: Partial + /** + * Used in single-tab mode to restart the video capture to a new file without relaunching the browser. + */ + restart: () => Promise + writeVideoFrame: WriteVideoFrame +} + +/** + * Interface passed to the browser to enable capturing video. + */ +export type RunModeVideoApi = { + onError: (err: Error) => void + videoName: string + compressedVideoName: string + /** + * Create+use a new VideoController that uses ffmpeg to stream frames from `writeVideoFrame` to disk. + */ + useFfmpegVideoController: (opts?: { webmInput?: boolean}) => Promise + /** + * Register a non-ffmpeg video controller. + */ + useVideoController: (videoController?: BrowserVideoController) => void + /** + * Registers a handler for project.on('capture:video:frames'). + */ + onProjectCaptureVideoFrames: (fn: (data: Buffer) => void) => void +} diff --git a/system-tests/lib/dep-installer/index.ts b/system-tests/lib/dep-installer/index.ts index e8aeaeefdb41..67babfbab1c4 100644 --- a/system-tests/lib/dep-installer/index.ts +++ b/system-tests/lib/dep-installer/index.ts @@ -291,6 +291,7 @@ export async function scaffoldCommonNodeModules () { 'jimp', 'lazy-ass', 'lodash', + 'playwright-webkit', 'proxyquire', 'semver', 'systeminformation', diff --git a/system-tests/lib/normalizeStdout.ts b/system-tests/lib/normalizeStdout.ts index 5d10e5eb43d7..1bac6aa44d2d 100644 --- a/system-tests/lib/normalizeStdout.ts +++ b/system-tests/lib/normalizeStdout.ts @@ -3,11 +3,11 @@ import _ from 'lodash' export const e2ePath = Fixtures.projectPath('e2e') -export const DEFAULT_BROWSERS = ['electron', 'chrome', 'firefox'] +export const DEFAULT_BROWSERS = ['electron', 'chrome', 'firefox', 'webkit'] export const pathUpToProjectName = Fixtures.projectPath('') -export const browserNameVersionRe = /(Browser\:\s+)(Custom |)(Electron|Chrome|Canary|Chromium|Firefox)(\s\d+)(\s\(\w+\))?(\s+)/ +export const browserNameVersionRe = /(Browser\:\s+)(Custom |)(Electron|Chrome|Canary|Chromium|Firefox|WebKit)(\s\d+)(\s\(\w+\))?(\s+)/ const stackTraceLinesRe = /(\n?[^\S\n\r]*).*?(@|\bat\b)(?:.*node:.*|.*\.(js|coffee|ts|html|jsx|tsx))\??(-\d+)?:\d+:\d+[\n\S\s]*?(\n\s*?\n|$)/g const availableBrowsersRe = /(Available browsers found on your system are:)([\s\S]+)/g diff --git a/system-tests/lib/system-tests.ts b/system-tests/lib/system-tests.ts index 2262716da5f1..1a94a240d3b5 100644 --- a/system-tests/lib/system-tests.ts +++ b/system-tests/lib/system-tests.ts @@ -40,8 +40,8 @@ require(`@packages/server/lib/project-base`) type CypressConfig = { [key: string]: any } -export type BrowserName = 'electron' | 'firefox' | 'chrome' -| '!electron' | '!chrome' | '!firefox' +export type BrowserName = 'electron' | 'firefox' | 'chrome' | 'webkit' +| '!electron' | '!chrome' | '!firefox' | '!webkit' type ExecResult = { code: number @@ -854,7 +854,7 @@ const systemTests = { const { browser } = options if (browser && !customBrowserPath) { - expect(_.capitalize(browser)).to.eq(browserName) + expect(String(browser).toLowerCase()).to.eq(browserName.toLowerCase()) } expect(parseFloat(version)).to.be.a.number diff --git a/system-tests/test/async_timeouts_spec.js b/system-tests/test/async_timeouts_spec.js index 857d38cc30b9..cc22f18bbe67 100644 --- a/system-tests/test/async_timeouts_spec.js +++ b/system-tests/test/async_timeouts_spec.js @@ -4,6 +4,7 @@ describe('e2e async timeouts', () => { systemTests.setup() systemTests.it('failing1', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'async_timeouts.cy.js', snapshot: true, expectedExitCode: 2, diff --git a/system-tests/test/before_browser_launch_spec.ts b/system-tests/test/before_browser_launch_spec.ts index 514362358dcf..0b1273296e48 100644 --- a/system-tests/test/before_browser_launch_spec.ts +++ b/system-tests/test/before_browser_launch_spec.ts @@ -26,6 +26,7 @@ describe('e2e before:browser:launch', () => { }) systemTests.it('can add extensions', { + browser: '!webkit', // TODO(webkit): fix+unskip, or skip and add a test that this fails with WebKit spec: 'spec.cy.js', config: { video: false, diff --git a/system-tests/test/caught_uncaught_hook_errors_spec.js b/system-tests/test/caught_uncaught_hook_errors_spec.js index 8831628bd0e7..934a4750467e 100644 --- a/system-tests/test/caught_uncaught_hook_errors_spec.js +++ b/system-tests/test/caught_uncaught_hook_errors_spec.js @@ -9,24 +9,28 @@ describe('e2e caught and uncaught hooks errors', () => { }) systemTests.it('failing1', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'hook_caught_error_failing.cy.js', snapshot: true, expectedExitCode: 3, }) systemTests.it('failing2', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to different ReferenceError snapshot) spec: 'hook_uncaught_error_failing.cy.js', snapshot: true, expectedExitCode: 1, }) systemTests.it('failing3', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to different ReferenceError snapshot) spec: 'hook_uncaught_root_error_failing.cy.js', snapshot: true, expectedExitCode: 1, }) systemTests.it('failing4', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to different ReferenceError snapshot) spec: 'hook_uncaught_error_events_failing.cy.js', snapshot: true, expectedExitCode: 1, diff --git a/system-tests/test/commands_outside_of_test_spec.js b/system-tests/test/commands_outside_of_test_spec.js index b7c2f62ad21a..60ff96c9edde 100644 --- a/system-tests/test/commands_outside_of_test_spec.js +++ b/system-tests/test/commands_outside_of_test_spec.js @@ -4,12 +4,14 @@ describe('e2e commands outside of test', () => { systemTests.setup() systemTests.it('fails on cy commands', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'commands_outside_of_test.cy.js', snapshot: true, expectedExitCode: 1, }) systemTests.it('fails on failing assertions', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'assertions_failing_outside_of_test.cy.js', snapshot: true, expectedExitCode: 1, diff --git a/system-tests/test/cookies_spec.ts b/system-tests/test/cookies_spec.ts index 05776b0e9281..5bf8bf8b248a 100644 --- a/system-tests/test/cookies_spec.ts +++ b/system-tests/test/cookies_spec.ts @@ -198,6 +198,7 @@ describe('e2e cookies', () => { // once browsers are shipping with the options in FORCED_SAMESITE_ENV as default, // we can remove this extra test case it('with forced SameSite strictness', { + browser: '!webkit', // TODO(webkit): fix+unskip config: { baseUrl, env: { @@ -251,6 +252,7 @@ describe('e2e cookies', () => { ], ) => { it(`passes with baseurl: ${baseUrl}`, { + browser: '!webkit', // TODO(webkit): fix+unskip config: { baseUrl, env: { @@ -274,6 +276,7 @@ describe('e2e cookies', () => { }) it('passes with no baseurl', { + browser: '!webkit', // TODO(webkit): fix+unskip config: { env: { httpUrl, @@ -347,6 +350,7 @@ describe('cross-origin cookies, set:cookies', () => { // https://github.com/cypress-io/cypress/issues/6375 it('set:cookies', { + browser: '!webkit', // TODO(webkit): fix+unskip (needs multidomain support) config: { video: false, baseUrl: `http://127.0.0.3:${httpPort}`, diff --git a/system-tests/test/cy_origin_error_spec.ts b/system-tests/test/cy_origin_error_spec.ts index ce43ac567393..a465a1be6ce9 100644 --- a/system-tests/test/cy_origin_error_spec.ts +++ b/system-tests/test/cy_origin_error_spec.ts @@ -28,6 +28,7 @@ describe('e2e cy.origin errors', () => { }) systemTests.it('captures the stack trace correctly for errors in cross origins to point users to their "cy.origin" callback', { + browser: '!webkit', // TODO(webkit): fix+unskip (needs multidomain support) // keep the port the same to prevent issues with the snapshot port: PORT, spec: 'cy_origin_error.cy.ts', diff --git a/system-tests/test/cy_origin_retries_spec.ts b/system-tests/test/cy_origin_retries_spec.ts index 88d8e27ea67e..0502b8e26a64 100644 --- a/system-tests/test/cy_origin_retries_spec.ts +++ b/system-tests/test/cy_origin_retries_spec.ts @@ -26,6 +26,7 @@ describe('e2e cy.origin retries', () => { }) systemTests.it('Appropriately displays test retry errors without other side effects', { + browser: '!webkit', // TODO(webkit): fix+unskip (needs multidomain support) // keep the port the same to prevent issues with the snapshot port: PORT, spec: 'cy_origin_retries.cy.ts', diff --git a/system-tests/test/deprecated_spec.ts b/system-tests/test/deprecated_spec.ts index 425f6e7e72a0..3402ca796df1 100644 --- a/system-tests/test/deprecated_spec.ts +++ b/system-tests/test/deprecated_spec.ts @@ -18,6 +18,7 @@ describe('deprecated before:browser:launch args', () => { systemTests.setup() systemTests.it('fails when adding unknown properties to launchOptions', { + browser: '!webkit', // TODO(webkit): fix+unskip (add executeBeforeBrowserLaunch to WebKit) config: { video: false, env: { @@ -31,6 +32,7 @@ describe('deprecated before:browser:launch args', () => { }) systemTests.it('push and no return - warns user exactly once', { + browser: '!webkit', // TODO(webkit): fix+unskip (add executeBeforeBrowserLaunch to WebKit) config: { video: false, env: { @@ -44,6 +46,7 @@ describe('deprecated before:browser:launch args', () => { }) systemTests.it('using non-deprecated API - no warning', { + browser: '!webkit', // TODO(webkit): fix+unskip (add executeBeforeBrowserLaunch to WebKit) // TODO: implement webPreferences.additionalArgs here // once we decide if/what we're going to make the implemenation // SUGGESTION: add this to Cypress.browser.args which will capture @@ -61,6 +64,7 @@ describe('deprecated before:browser:launch args', () => { }) systemTests.it('concat return returns once', { + browser: '!webkit', // TODO(webkit): fix+unskip (add executeBeforeBrowserLaunch to WebKit) // TODO: implement webPreferences.additionalArgs here // once we decide if/what we're going to make the implemenation // SUGGESTION: add this to Cypress.browser.args which will capture @@ -109,6 +113,7 @@ describe('deprecated before:browser:launch args', () => { // printed. we should print that we are aborting the run because // the before:browser:launch handler threw an error / rejected systemTests.it('displays errors thrown and aborts the run', { + browser: '!webkit', // TODO(webkit): fix+unskip (add executeBeforeBrowserLaunch to WebKit) config: { video: false, env: { @@ -127,6 +132,7 @@ describe('deprecated before:browser:launch args', () => { // printed. we should print that we are aborting the run because // the before:browser:launch handler threw an error / rejected systemTests.it('displays promises rejected and aborts the run', { + browser: '!webkit', // TODO(webkit): fix+unskip (add executeBeforeBrowserLaunch to WebKit) config: { video: false, env: { diff --git a/system-tests/test/downloads_spec.ts b/system-tests/test/downloads_spec.ts index 03ee533e4a49..8c7fc6449967 100644 --- a/system-tests/test/downloads_spec.ts +++ b/system-tests/test/downloads_spec.ts @@ -10,6 +10,7 @@ describe('e2e downloads', () => { systemTests.setup() systemTests.it('handles various file downloads', { + browser: '!webkit', // TODO(webkit): fix+unskip (implement downloads support) project: 'downloads', spec: 'downloads.cy.ts', config: { @@ -22,6 +23,7 @@ describe('e2e downloads', () => { } systemTests.it('allows changing the downloads folder', { + browser: '!webkit', // TODO(webkit): fix+unskip (implement downloads support) project: 'downloads', spec: 'downloads.cy.ts', config: { diff --git a/system-tests/test/error_ui_spec.ts b/system-tests/test/error_ui_spec.ts index 8fa8bebc7b09..488667ab291f 100644 --- a/system-tests/test/error_ui_spec.ts +++ b/system-tests/test/error_ui_spec.ts @@ -24,6 +24,7 @@ describe('e2e error ui', function () { ] .forEach((project) => { systemTests.it(`handles sourcemaps in webpack for project: ${project}`, { + browser: '!webkit', // TODO(webkit): fix+unskip project, spec: 'failing.*', expectedExitCode: 1, @@ -35,6 +36,7 @@ describe('e2e error ui', function () { // https://github.com/cypress-io/cypress/issues/16255 systemTests.it('handles errors when test files are outside of project root', { + browser: '!webkit', // TODO(webkit): fix+unskip project: 'integration-outside-project-root/project-root', spec: '../../../e2e/failing.cy.js', expectedExitCode: 1, diff --git a/system-tests/test/fetch_polyfill_spec.js b/system-tests/test/fetch_polyfill_spec.js index e7ff0481e733..e70c938a971e 100644 --- a/system-tests/test/fetch_polyfill_spec.js +++ b/system-tests/test/fetch_polyfill_spec.js @@ -136,6 +136,7 @@ describe('e2e fetch polyfill', () => { }) systemTests.it('passes', { + browser: '!webkit', // TODO(webkit): fix+unskip spec: 'fetch.cy.js', snapshot: false, config: { diff --git a/system-tests/test/form_submissions_spec.js b/system-tests/test/form_submissions_spec.js index 701cc37b62b9..25f6c990fa2b 100644 --- a/system-tests/test/form_submissions_spec.js +++ b/system-tests/test/form_submissions_spec.js @@ -95,6 +95,7 @@ describe('e2e forms', () => { }) systemTests.it('failing', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'form_submission_failing.cy.js', snapshot: true, expectedExitCode: 1, diff --git a/system-tests/test/headed_spec.ts b/system-tests/test/headed_spec.ts index 5d01cb2f3060..cf636c08b2f0 100644 --- a/system-tests/test/headed_spec.ts +++ b/system-tests/test/headed_spec.ts @@ -1,19 +1,12 @@ -import systemTests, { BrowserName } from '../lib/system-tests' +import systemTests from '../lib/system-tests' describe('e2e headed', function () { systemTests.setup() - const browserList: BrowserName[] = ['chrome', 'firefox', 'electron'] - - browserList.forEach(function (browser) { - it(`runs multiple specs in headed mode - [${browser}]`, async function () { - await systemTests.exec(this, { - project: 'e2e', - headed: true, - browser, - spec: 'a_record.cy.js,b_record.cy.js,simple_passing.cy.js', - expectedExitCode: 0, - }) - }) + systemTests.it(`runs multiple specs in headed mode`, { + project: 'e2e', + headed: true, + spec: 'a_record.cy.js,b_record.cy.js,simple_passing.cy.js', + expectedExitCode: 0, }) }) diff --git a/system-tests/test/issue_173_spec.ts b/system-tests/test/issue_173_spec.ts index 8973f636f4bd..b7b3891f8a75 100644 --- a/system-tests/test/issue_173_spec.ts +++ b/system-tests/test/issue_173_spec.ts @@ -5,6 +5,7 @@ describe('e2e issue 173', () => { // https://github.com/cypress-io/cypress/issues/173 systemTests.it('failing', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'issue_173.cy.js', snapshot: true, expectedExitCode: 1, diff --git a/system-tests/test/issue_674_spec.js b/system-tests/test/issue_674_spec.js index 3cb7be05f2b2..ab2155f9aa2f 100644 --- a/system-tests/test/issue_674_spec.js +++ b/system-tests/test/issue_674_spec.js @@ -5,6 +5,7 @@ describe('e2e issue 674', () => { // https://github.com/cypress-io/cypress/issues/674 systemTests.it('fails', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'issue_674.cy.js', snapshot: true, expectedExitCode: 1, diff --git a/system-tests/test/js_error_handling_spec.js b/system-tests/test/js_error_handling_spec.js index 9807c776e8ce..503ae72959b9 100644 --- a/system-tests/test/js_error_handling_spec.js +++ b/system-tests/test/js_error_handling_spec.js @@ -46,6 +46,7 @@ describe('e2e js error handling', () => { }) systemTests.it('fails', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'js_error_handling_failing.cy.js', snapshot: true, expectedExitCode: 5, diff --git a/system-tests/test/page_loading_spec.js b/system-tests/test/page_loading_spec.js index 7d3f534ab01c..c1b4ba65c1db 100644 --- a/system-tests/test/page_loading_spec.js +++ b/system-tests/test/page_loading_spec.js @@ -78,6 +78,7 @@ describe('e2e page_loading', () => { // set we send an XHR which should not inject because its requested for JSON // but that another XHR which is requested for html should inject systemTests.it('passes', { + browser: '!webkit', // TODO(webkit): fix+unskip (related to document.cookie issue?) spec: 'page_loading.cy.js', snapshot: true, }) diff --git a/system-tests/test/promises_spec.js b/system-tests/test/promises_spec.js index da52bf0f143e..f468b56e5428 100644 --- a/system-tests/test/promises_spec.js +++ b/system-tests/test/promises_spec.js @@ -4,6 +4,7 @@ describe('e2e promises', () => { systemTests.setup() systemTests.it('failing1', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'promises.cy.js', snapshot: true, expectedExitCode: 2, diff --git a/system-tests/test/request_spec.ts b/system-tests/test/request_spec.ts index 2fb64dfe5342..90fd26447c67 100644 --- a/system-tests/test/request_spec.ts +++ b/system-tests/test/request_spec.ts @@ -168,6 +168,7 @@ describe('e2e requests', () => { }) systemTests.it('passes', { + browser: '!webkit', // TODO(webkit): fix+unskip spec: 'request.cy.js', snapshot: true, config: { diff --git a/system-tests/test/runnable_execution_spec.ts b/system-tests/test/runnable_execution_spec.ts index a30e14c7c82a..62a22c67dd99 100644 --- a/system-tests/test/runnable_execution_spec.ts +++ b/system-tests/test/runnable_execution_spec.ts @@ -20,6 +20,7 @@ describe('e2e runnable execution', () => { // but throws correct error // https://github.com/cypress-io/cypress/issues/1987 systemTests.it('cannot navigate in before hook and test', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) project: 'hooks-after-rerun', spec: 'beforehook-and-test-navigation.cy.js', snapshot: true, @@ -33,6 +34,7 @@ describe('e2e runnable execution', () => { }) systemTests.it('runs correctly after top navigation with already ran suite', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'runnables_already_run_suite.cy.js', snapshot: true, expectedExitCode: 1, diff --git a/system-tests/test/screenshots_spec.js b/system-tests/test/screenshots_spec.js index d62583db35dc..1d24bd76e141 100644 --- a/system-tests/test/screenshots_spec.js +++ b/system-tests/test/screenshots_spec.js @@ -61,6 +61,7 @@ describe('e2e screenshots', () => { // and are also generated automatically on failure with // the test title as the file name systemTests.it('passes', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing partially due to broken stack trace) spec: 'screenshots.cy.js', expectedExitCode: 5, snapshot: true, diff --git a/system-tests/test/server_sent_events_spec.js b/system-tests/test/server_sent_events_spec.js index f237e5da5521..9a130a96a65d 100644 --- a/system-tests/test/server_sent_events_spec.js +++ b/system-tests/test/server_sent_events_spec.js @@ -67,6 +67,7 @@ describe('e2e server sent events', () => { // https://github.com/cypress-io/cypress/issues/1440 systemTests.it('passes', { + browser: '!webkit', // TODO(webkit): fix+unskip spec: 'server_sent_events.cy.js', snapshot: true, }) diff --git a/system-tests/test/session_spec.ts b/system-tests/test/session_spec.ts index 20e3003c9411..493a4b808f9d 100644 --- a/system-tests/test/session_spec.ts +++ b/system-tests/test/session_spec.ts @@ -131,6 +131,7 @@ describe('e2e sessions', () => { }) it('session tests', { + browser: '!webkit', // TODO(webkit): fix+unskip (needs multidomain support) spec: 'session.cy.js', snapshot: true, config: { @@ -140,6 +141,7 @@ describe('e2e sessions', () => { }) it('sessions persist on reload, and clear between specs', { + browser: '!webkit', // TODO(webkit): fix+unskip (needs multidomain support) spec: 'session_persist_spec_1.cy.js,session_persist_spec_2.cy.js', snapshot: true, config: { @@ -149,6 +151,7 @@ describe('e2e sessions', () => { }) it('sessions recreated on reload in open mode', { + browser: '!webkit', // TODO(webkit): fix+unskip (needs multidomain support) spec: 'session_recreate_reload.cy.js', snapshot: true, config: { diff --git a/system-tests/test/spec_isolation_spec.js b/system-tests/test/spec_isolation_spec.js index 315952d6005c..643da5056210 100644 --- a/system-tests/test/spec_isolation_spec.js +++ b/system-tests/test/spec_isolation_spec.js @@ -21,6 +21,7 @@ describe('e2e spec_isolation', () => { systemTests.setup() it('fails', { + browser: '!webkit', // TODO(webkit): fix+unskip spec: specs, outputPath, snapshot: false, @@ -51,6 +52,7 @@ describe('e2e spec_isolation', () => { }) it('failing with retries enabled', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'simple_failing_hook.cy.js,simple_retrying.cy.js', outputPath, snapshot: true, diff --git a/system-tests/test/stdout_spec.js b/system-tests/test/stdout_spec.js index 857c827d5155..d45474d43c8a 100644 --- a/system-tests/test/stdout_spec.js +++ b/system-tests/test/stdout_spec.js @@ -65,6 +65,7 @@ describe('e2e stdout', () => { }) systemTests.it('displays assertion errors', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'stdout_assertion_errors.cy.js', snapshot: true, expectedExitCode: 4, diff --git a/system-tests/test/subdomain_spec.ts b/system-tests/test/subdomain_spec.ts index 1d2d8f2fabb8..0abcf7991403 100644 --- a/system-tests/test/subdomain_spec.ts +++ b/system-tests/test/subdomain_spec.ts @@ -119,6 +119,7 @@ describe('e2e subdomain', () => { }) systemTests.it('passes', { + browser: '!webkit', // TODO(webkit): fix+unskip spec: 'subdomain.cy.js', snapshot: true, config: { diff --git a/system-tests/test/testConfigOverrides_spec.ts b/system-tests/test/testConfigOverrides_spec.ts index 429286af549c..30ee1fc8c6fa 100644 --- a/system-tests/test/testConfigOverrides_spec.ts +++ b/system-tests/test/testConfigOverrides_spec.ts @@ -11,6 +11,7 @@ describe('testConfigOverrides', () => { systemTests.setup() systemTests.it('fails when passing invalid config value browser', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'testConfigOverrides/invalid-browser.js', snapshot: true, expectedExitCode: 1, diff --git a/system-tests/test/uncaught_spec_errors_spec.js b/system-tests/test/uncaught_spec_errors_spec.js index 1117a537ff97..72199ce7c953 100644 --- a/system-tests/test/uncaught_spec_errors_spec.js +++ b/system-tests/test/uncaught_spec_errors_spec.js @@ -4,30 +4,35 @@ describe('e2e uncaught errors', () => { systemTests.setup() systemTests.it('failing1', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'uncaught_synchronous_before_tests_parsed.js', snapshot: true, expectedExitCode: 1, }) systemTests.it('failing2', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'uncaught_synchronous_during_hook.cy.js', snapshot: true, expectedExitCode: 1, }) systemTests.it('failing3', { + browser: '!webkit', // TODO(webkit): fix+unskip spec: 'uncaught_during_test.cy.js', snapshot: true, expectedExitCode: 3, }) systemTests.it('failing4', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'uncaught_during_hook.cy.js', snapshot: true, expectedExitCode: 1, }) systemTests.it('failing5', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'caught_async_sync_test.cy.js', snapshot: true, expectedExitCode: 4, diff --git a/system-tests/test/user_agent_spec.js b/system-tests/test/user_agent_spec.js index fcf36431b10d..7f0f04120257 100644 --- a/system-tests/test/user_agent_spec.js +++ b/system-tests/test/user_agent_spec.js @@ -29,6 +29,7 @@ describe('e2e user agent', () => { }) systemTests.it('passes', { + browser: '!webkit', // TODO(webkit): fix+unskip spec: 'user_agent.cy.js', snapshot: true, }) diff --git a/system-tests/test/visit_spec.js b/system-tests/test/visit_spec.js index 6609fb03a8c8..145882546871 100644 --- a/system-tests/test/visit_spec.js +++ b/system-tests/test/visit_spec.js @@ -131,6 +131,7 @@ describe('e2e visit', () => { }) systemTests.it('passes', { + browser: '!webkit', // TODO(webkit): fix+unskip spec: 'visit.cy.js', snapshot: true, onRun (exec) { @@ -145,6 +146,7 @@ describe('e2e visit', () => { }) systemTests.it('passes with experimentalSourceRewriting', { + browser: '!webkit', // TODO(webkit): fix+unskip spec: 'source_rewriting.cy.js', config: { experimentalSourceRewriting: true, @@ -169,6 +171,7 @@ describe('e2e visit', () => { }) systemTests.it('fails when server responds with 500', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'visit_http_500_response_failing.cy.js', snapshot: true, expectedExitCode: 1, @@ -182,6 +185,7 @@ describe('e2e visit', () => { }) systemTests.it('fails when content type isnt html', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'visit_non_html_content_type_failing.cy.js', snapshot: true, expectedExitCode: 1, @@ -207,6 +211,7 @@ describe('e2e visit', () => { }) systemTests.it('fails when response never ends', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'visit_response_never_ends_failing.cy.js', snapshot: true, expectedExitCode: 3, @@ -227,6 +232,7 @@ describe('e2e visit', () => { }) systemTests.it('fails when visit times out', { + browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'visit_http_timeout_failing.cy.js', snapshot: true, expectedExitCode: 2, diff --git a/system-tests/test/web_security_spec.js b/system-tests/test/web_security_spec.js index 4b9805329797..4cd70cba4f9a 100644 --- a/system-tests/test/web_security_spec.js +++ b/system-tests/test/web_security_spec.js @@ -72,6 +72,7 @@ describe('e2e web security', () => { context('when enabled', () => { systemTests.it('fails', { + browser: '!webkit', // TODO(webkit): fix+unskip spec: 'web_security.cy.js', config: { experimentalSessionAndOrigin: true, @@ -98,6 +99,7 @@ describe('e2e web security', () => { systemTests.it('displays warning when firefox and chromeWebSecurity:false', { spec: 'simple_passing.cy.js', snapshot: true, + // TODO(webkit): run this test in webkit browser: 'firefox', config: { chromeWebSecurity: false, diff --git a/system-tests/test/websockets_spec.js b/system-tests/test/websockets_spec.js index 5f64b72120e6..8a41e7fa1ed8 100644 --- a/system-tests/test/websockets_spec.js +++ b/system-tests/test/websockets_spec.js @@ -37,6 +37,7 @@ describe('e2e websockets', () => { // https://github.com/cypress-io/cypress/issues/556 systemTests.it('passes', { + browser: '!webkit', // TODO(webkit): fix+unskip spec: 'websockets.cy.js', snapshot: true, }) diff --git a/system-tests/test/xhr_spec.js b/system-tests/test/xhr_spec.js index 12438445861a..f5b753899ca1 100644 --- a/system-tests/test/xhr_spec.js +++ b/system-tests/test/xhr_spec.js @@ -29,11 +29,13 @@ describe('e2e xhr', () => { }) systemTests.it('passes in global mode', { + browser: '!webkit', // TODO(webkit): fix+unskip spec: 'xhr.cy.js', snapshot: true, }) systemTests.it('passes through CLI', { + browser: '!webkit', // TODO(webkit): fix+unskip spec: 'xhr.cy.js', snapshot: true, useCli: true,