feat: Adding a cache layer for remote queries, e2e test helpers (#18652)

Author: tgriesser

packages/data-context/package.json

@@ -13,14 +13,18 @@
     "test-integration": "mocha -r @packages/ts/register test/integration/**/*.spec.ts --config ./test/.mocharc.js --exit"
   },
   "dependencies": {
-    "dataloader": "^2.0.0"
+    "@storybook/csf-tools": "^6.4.0-alpha.38",
+    "cross-fetch": "^3.1.4",
+    "dataloader": "^2.0.0",
+    "globby": "^11.0.1",
+    "lodash": "4.17.21",
+    "p-defer": "^3.0.0",
+    "wonka": "^4.0.15"
   },
   "devDependencies": {
     "@packages/ts": "0.0.0-development",
     "@packages/types": "0.0.0-development",
-    "@storybook/csf-tools": "^6.4.0-alpha.38",
     "create-cypress-tests": "0.0.0-development",
-    "globby": "^11.0.1",
     "mocha": "7.0.1",
     "rimraf": "3.0.2"
   },

packages/data-context/src/DataContext.ts

@@ -16,6 +16,7 @@ import {
   BrowserDataSource,
   UtilDataSource,
   StorybookDataSource,
+  CloudDataSource,
 } from './sources/'
 import { cached } from './util/cached'
 import { DataContextShell, DataContextShellConfig } from './DataContextShell'
@@ -63,6 +64,8 @@ export class DataContext extends DataContextShell {
       this.actions.app.refreshBrowsers(),
       // load projects from cache on start
       this.actions.project.loadProjects(),
+      // load the cached user & validate the token on start
+      this.actions.auth.getUser(),
     ]
 
     if (this.config.launchArgs.projectRoot) {
@@ -172,6 +175,11 @@ export class DataContext extends DataContextShell {
     return new ProjectDataSource(this)
   }
 
+  @cached
+  get cloud () {
+    return new CloudDataSource(this)
+  }
+
   get projectsList () {
     return this.coreData.app.projects
   }
@@ -210,6 +218,16 @@ export class DataContext extends DataContextShell {
     console.error(e)
   }
 
+  /**
+   * If we really want to get around the guards added in proxyContext
+   * which disallow referencing ctx.actions / ctx.emitter from contexct for a GraphQL query,
+   * we can call ctx.deref.emitter, etc. This should only be used in exceptional situations where
+   * we're certain this is a good idea.
+   */
+  get deref () {
+    return this
+  }
+
   async destroy () {
     super.destroy()
 

packages/data-context/src/actions/AuthActions.ts

@@ -3,20 +3,40 @@ import type { AuthenticatedUserShape } from '../data'
 
 interface AuthMessage {type: string, browserOpened: boolean, name: string, message: string}
 export interface AuthApiShape {
+  getUser(): Promise<Partial<AuthenticatedUserShape>>
   logIn(onMessage: (message: AuthMessage) => void): Promise<AuthenticatedUserShape>
   logOut(): Promise<void>
-  checkAuth(context: DataContext): Promise<void>
 }
 
 export class AuthActions {
   constructor (private ctx: DataContext) {}
 
+  async getUser () {
+    return this.authApi.getUser().then((obj) => {
+      if (obj.authToken) {
+        this.ctx.coreData.user = obj
+        // When we get the user at startup, check the auth by
+        // hitting the network
+        this.checkAuth()
+      }
+    })
+  }
+
   get authApi () {
     return this.ctx._apis.authApi
   }
 
   async checkAuth () {
-    return this.authApi.checkAuth(this.ctx)
+    const result = await this.ctx.cloud.executeRemoteGraphQL({
+      query: `{ cloudViewer { id } }`,
+      variables: {},
+      requestPolicy: 'network-only',
+    })
+
+    if (!result.data?.cloudViewer) {
+      this.ctx.coreData.user = null
+      this.logout()
+    }
   }
 
   async login () {

packages/data-context/src/sources/CloudDataSource.ts

@@ -0,0 +1,93 @@
+import type { DataContext } from '..'
+import pDefer from 'p-defer'
+import getenv from 'getenv'
+import { pipe, subscribe, toPromise } from 'wonka'
+import type { DocumentNode } from 'graphql'
+import {
+  createClient,
+  cacheExchange,
+  dedupExchange,
+  fetchExchange,
+  Client,
+  createRequest,
+  OperationResult,
+  RequestPolicy,
+} from '@urql/core'
+import _ from 'lodash'
+
+const cloudEnv = getenv('CYPRESS_INTERNAL_CLOUD_ENV', process.env.CYPRESS_INTERNAL_ENV || 'development') as keyof typeof REMOTE_SCHEMA_URLS
+
+const REMOTE_SCHEMA_URLS = {
+  staging: 'https://dashboard-staging.cypress.io',
+  development: 'http://localhost:3000',
+  production: 'https://dashboard.cypress.io',
+}
+
+export interface CloudExecuteRemote {
+  query: string
+  document?: DocumentNode
+  variables: any
+  requestPolicy?: RequestPolicy
+}
+
+export class CloudDataSource {
+  private _urqlClient: Client
+
+  constructor (private ctx: DataContext) {
+    this._urqlClient = createClient({
+      url: `${REMOTE_SCHEMA_URLS[cloudEnv]}/test-runner-graphql`,
+      exchanges: [
+        dedupExchange,
+        cacheExchange,
+        fetchExchange,
+      ],
+      fetch: this.ctx.util.fetch,
+    })
+  }
+
+  async executeRemoteGraphQL (config: CloudExecuteRemote): Promise<Partial<OperationResult>> {
+    if (!this.ctx.user) {
+      return { data: null }
+    }
+
+    const requestPolicy = config.requestPolicy ?? 'cache-and-network'
+
+    const executingQuery = this._urqlClient.executeQuery(createRequest(config.query, config.variables), {
+      fetch: this.ctx.util.fetch,
+      requestPolicy,
+      fetchOptions: {
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': `bearer ${this.ctx.user.authToken}`,
+        },
+      },
+    })
+
+    if (requestPolicy === 'cache-and-network') {
+      let resolvedData: OperationResult | undefined = undefined
+      const dfd = pDefer<OperationResult>()
+      const pipeline = pipe(
+        executingQuery,
+        subscribe((res) => {
+          if (!resolvedData) {
+            resolvedData = res
+            dfd.resolve(res)
+          } else if (!_.isEqual(resolvedData.data, res.data) || !_.isEqual(resolvedData.error, res.error)) {
+            // TODO(tim): send a signal to the frontend so when it refetches it does 'cache-only' request,
+            // since we know we're up-to-date
+            this.ctx.deref.emitter.toApp()
+            this.ctx.deref.emitter.toLaunchpad()
+          }
+
+          if (!res.stale) {
+            pipeline.unsubscribe()
+          }
+        }),
+      )
+
+      return dfd.promise
+    }
+
+    return pipe(executingQuery, toPromise)
+  }
+}

packages/data-context/src/sources/UtilDataSource.ts

@@ -1,4 +1,7 @@
 import DataLoader from 'dataloader'
+import crypto from 'crypto'
+import fetch from 'cross-fetch'
+
 import type { DataContext } from '..'
 
 /**
@@ -41,4 +44,12 @@ export class UtilDataSource {
       loader.clearAll()
     }
   }
+
+  sha1 (value: string) {
+    return crypto.createHash('sha1').update(value).digest('hex')
+  }
+
+  get fetch () {
+    return fetch
+  }
 }

packages/data-context/src/sources/index.ts

@@ -3,6 +3,7 @@
 
 export * from './AppDataSource'
 export * from './BrowserDataSource'
+export * from './CloudDataSource'
 export * from './FileDataSource'
 export * from './GitDataSource'
 export * from './ProjectDataSource'

packages/frontend-shared/cypress/e2e/e2ePluginSetup.ts

@@ -1,24 +1,33 @@
+import path from 'path'
+import type { RemoteGraphQLInterceptor, WithCtxInjected, WithCtxOptions } from './support/e2eSupport'
+import { e2eProjectDirs } from './support/e2eProjectDirs'
+import type { CloudExecuteRemote } from '@packages/data-context/src/sources'
 import type { DataContext } from '@packages/data-context'
 import * as inspector from 'inspector'
 import sinonChai from '@cypress/sinon-chai'
 import sinon from 'sinon'
 import rimraf from 'rimraf'
 import util from 'util'
+import fs from 'fs'
+import { buildSchema, execute, parse } from 'graphql'
+import { Response } from 'cross-fetch'
+
+import { CloudRunQuery } from '../support/mock-graphql/stubgql-CloudTypes'
+import { getOperationName } from '@urql/core'
+
+const cloudSchema = buildSchema(fs.readFileSync(path.join(__dirname, '../../../graphql/schemas/cloud.graphql'), 'utf8'))
 
 // require'd so we don't conflict with globals loaded in @packages/types
 const chai = require('chai')
 const chaiAsPromised = require('chai-as-promised')
 const chaiSubset = require('chai-subset')
+
 const { expect } = chai
 
 chai.use(chaiAsPromised)
 chai.use(chaiSubset)
 chai.use(sinonChai)
 
-import path from 'path'
-import type { WithCtxInjected, WithCtxOptions } from './support/e2eSupport'
-import { e2eProjectDirs } from './support/e2eProjectDirs'
-
 export async function e2ePluginSetup (projectRoot: string, on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) {
   process.env.CYPRESS_INTERNAL_E2E_TESTING_SELF = 'true'
   // require'd so we don't import the types from @packages/server which would
@@ -35,14 +44,22 @@ export async function e2ePluginSetup (projectRoot: string, on: Cypress.PluginEve
     fn: string
     options: WithCtxOptions
     activeTestId: string
+    // TODO(tim): add an API for intercepting this
+    interceptCloudExecute: (config: CloudExecuteRemote) => {}
   }
 
   let ctx: DataContext
   let serverPortPromise: Promise<number>
   let currentTestId: string | undefined
   let testState: Record<string, any> = {}
+  let remoteGraphQLIntercept: RemoteGraphQLInterceptor | undefined
 
   on('task', {
+    remoteGraphQLIntercept (fn: string) {
+      remoteGraphQLIntercept = new Function('console', 'obj', `return (${fn})(obj)`).bind(null, console) as RemoteGraphQLInterceptor
+
+      return null
+    },
     setupE2E (projectName: string) {
       Fixtures.scaffoldProject(projectName)
 
@@ -53,11 +70,49 @@ export async function e2ePluginSetup (projectRoot: string, on: Cypress.PluginEve
       if (obj.activeTestId !== currentTestId) {
         await ctx?.destroy()
         currentTestId = obj.activeTestId
+        remoteGraphQLIntercept = undefined
         testState = {};
         ({ serverPortPromise, ctx } = runInternalServer({
           projectRoot: null,
         }) as {ctx: DataContext, serverPortPromise: Promise<number>})
 
+        const fetchApi = ctx.util.fetch
+
+        sinon.stub(ctx.util, 'fetch').get(() => {
+          return async (url: RequestInfo, init?: RequestInit) => {
+            if (!String(url).endsWith('/test-runner-graphql')) {
+              return fetchApi(url, init)
+            }
+
+            const { query, variables } = JSON.parse(String(init?.body))
+            const document = parse(query)
+            const operationName = getOperationName(document)
+
+            let result = await execute({
+              operationName,
+              document,
+              variableValues: variables,
+              schema: cloudSchema,
+              rootValue: CloudRunQuery,
+              contextValue: {
+                __server__: ctx,
+              },
+            })
+
+            if (remoteGraphQLIntercept) {
+              result = remoteGraphQLIntercept({
+                operationName,
+                variables,
+                document,
+                query,
+                result,
+              })
+            }
+
+            return new Response(JSON.stringify(result), { status: 200 })
+          }
+        })
+
         await serverPortPromise
       }