Express server (#4987)

* add motion

* fetcher with operation

* operation

* use the new fetchers

* split storage

* active operation divs

* cleanup operations

* types

* op with key

* use v3_fetcherPersist

* no need to check null

* separate operation component

* wip

* correct state flow

* remove debug

* remove wait

* adjust cleanup

* add spacing

* split hooks

* more efficient hooks

* comment

* more comment

* easier to read

* simplify a bit

* a much simpler hook

* split

* document the code and split it

* 6 digits

* fix color

* remove unused

* move down

* fix project passing

* adjust assertion

* getOperationDescription

* unless

* wip

* add env vars

* clean up deps

* i'll still call it "cool proxy"

* remove unneeded dep

* use originalrequest's proto

* express template + integrate with remix and hot reload

* pfft

* comment and section

* ascii banner

* remove line ref

* fallback for auth0

* add routes

* update ports and proxied routes

* update route

* fix(all) locally run express on 8000 and 8001, and haskell on 8002

* comment out

* remove

* fix equality check

* fix links

* rename

* fix finish flow

* finish mirroring routes

* add monitoring route

* adjust default headers

* revert pointless change

* clean up

* Update utopia-remix/readme.md

Co-authored-by: RheeseyB <1044774+Rheeseyb@users.noreply.github.com>

* Update utopia-remix/readme.md

Co-authored-by: RheeseyB <1044774+Rheeseyb@users.noreply.github.com>

* Update utopia-remix/readme.md

Co-authored-by: RheeseyB <1044774+Rheeseyb@users.noreply.github.com>

* update validators

* use let

* pin dev deps

---------

Co-authored-by: RheeseyB <1044774+Rheeseyb@users.noreply.github.com>

Author: ruggi

server/src/Utopia/Web/Executors/Common.hs

@@ -95,7 +95,7 @@ type Stop = IO ()
 data EnvironmentRuntime r = EnvironmentRuntime
   { _initialiseResources :: IO r
   , _startup             :: r -> IO Stop
-  , _envServerPort       :: r -> [Int]
+  , _envServerPort       :: r -> Int
   , _serverAPI           :: r -> Server API
   , _startupLogging      :: r -> Bool
   , _getLogger           :: r -> FastLogger
@@ -139,7 +139,7 @@ logoutOfSession sessionState cookie pageContents action = do
 portFromEnvironment :: IO Int
 portFromEnvironment = do
   fromEnvironment <- lookupEnv "PORT"
-  let portForEndpoint = fromMaybe 8000 $ do
+  let portForEndpoint = fromMaybe 8002 $ do
         envPort <- fromEnvironment
         readMaybe envPort
   return portForEndpoint

server/src/Utopia/Web/Executors/Development.hs

@@ -521,8 +521,8 @@ startup DevServerResources{..} = do
         killThread hashedFilenamesThread
         destroyAllResources _projectPool
 
-serverPortFromResources :: DevServerResources -> [Int]
-serverPortFromResources resources = [_serverPort resources, _serverPort resources + 1]
+serverPortFromResources :: DevServerResources -> Int
+serverPortFromResources resources = _serverPort resources
 
 shouldProxyWebpack :: IO Bool
 shouldProxyWebpack = do

server/src/Utopia/Web/Executors/Production.hs

@@ -467,8 +467,8 @@ startup ProductionServerResources{..} = do
         killThread hashedFilenamesThread
         destroyAllResources _projectPool
 
-serverPortFromResources :: ProductionServerResources -> [Int]
-serverPortFromResources resources = [_serverPort resources]
+serverPortFromResources :: ProductionServerResources -> Int
+serverPortFromResources resources = _serverPort resources
 
 productionEnvironmentRuntime :: EnvironmentRuntime ProductionServerResources
 productionEnvironmentRuntime = EnvironmentRuntime

server/src/Utopia/Web/Server.hs

@@ -176,14 +176,13 @@ runServerWithResources EnvironmentRuntime{..} = do
   when loggingEnabled $ loggerLn logger "Starting"
   shutdown <- _startup resources
   when loggingEnabled $ loggerLn logger "Startup Processes Completed"
-  let ports = _envServerPort resources
+  let port = _envServerPort resources
   when loggingEnabled $ do
-    forM_ ports $ \port -> do
-      -- Note: '<>' is used to append text (amongst other things).
-      loggerLn logger ("Running On: http://localhost:" <> toLogStr port <> "/")
+    -- Note: '<>' is used to append text (amongst other things).
+    loggerLn logger ("Running On: http://localhost:" <> toLogStr port <> "/")
   let storeForMetrics = _metricsStore resources
   meterMap <- mkMeterMap apiProxy storeForMetrics
-  let settingsList = [ Warp.setPort port $ Warp.setOnException (exceptionHandler (loggerLn logger)) Warp.defaultSettings | port <- ports ]
+  let settingsList = [Warp.setPort port $ Warp.setOnException (exceptionHandler (loggerLn logger)) Warp.defaultSettings]
   let serverAPI = _serverAPI resources
   let assetsCache = _cacheForAssets resources
   let shouldForceSSL = _forceSSL resources

shell.nix

@@ -554,7 +554,7 @@ let
       cd $(${pkgs.git}/bin/git rev-parse --show-toplevel)/utopia-remix
       ${pnpm}/bin/pnpm install
       ${pnpm}/bin/pnpm exec prisma generate
-      PORT=8002 ${pnpm}/bin/pnpm run dev
+      PORT=8000 ${pnpm}/bin/pnpm run dev
     '')
   ];
 

utopia-remix/.env.sample

@@ -1,5 +1,5 @@
 APP_ENV="local"
-BACKEND_URL="http://127.0.0.1:8001"
+BACKEND_URL="http://127.0.0.1:8002"
 CORS_ORIGIN="http://localhost:8000"
 DATABASE_URL="postgres://<username>:postgres@localhost:5432/utopia"
 REACT_APP_EDITOR_URL="http://localhost:8000"
@@ -8,14 +8,17 @@ AUTH0_ENDPOINT=""
 AUTH0_CLIENT_ID=""
 AUTH0_REDIRECT_URI=""
 
+GITHUB_OAUTH_CLIENT_ID=""
+GITHUB_OAUTH_REDIRECT_URL=""
+
+UTOPIA_CDN_URL="http://localhost:8000"
+
 FGA_STORE_ID=""
 FGA_CLIENT_ID=""
 FGA_SECRET=""
 FGA_API_HOST="api.us1.fga.dev"
 FGA_API_TOKEN_ISSUER="fga.us.auth0.com"
 FGA_API_AUDIENCE="https://api.us1.fga.dev/"
 
-GITHUB_OAUTH_CLIENT_ID=""
-GITHUB_OAUTH_REDIRECT_URL=""
-
-UTOPIA_CDN_URL="http://localhost:8000"
+EXPRESS_PROXY_TARGET_HOST="localhost"
+EXPRESS_PROXY_TARGET_PORT="8002"

utopia-remix/app/routes/monitoring.secret.location.tsx

@@ -0,0 +1,12 @@
+import { LoaderFunctionArgs } from '@remix-run/node'
+import { proxy } from '../util/proxy.server'
+import { handle, handleOptions } from '../util/api.server'
+import { ALLOW } from '../handlers/validators'
+
+// This is for monitoring/debugging the HS server, exposing it for added convenience.
+export async function loader(args: LoaderFunctionArgs) {
+  return handle(args, {
+    OPTIONS: handleOptions,
+    GET: { handler: proxy, validator: ALLOW },
+  })
+}

utopia-remix/app/routes/v1.asset.$id.assets.$name.tsx

@@ -1,7 +1,8 @@
 import { ActionFunctionArgs, LoaderFunctionArgs } from '@remix-run/node'
 import { proxy } from '../util/proxy.server'
 import { handle, handleOptions } from '../util/api.server'
-import { ALLOW } from '../handlers/validators'
+import { ALLOW, validateProjectAccess } from '../handlers/validators'
+import { UserProjectPermission } from '../types'
 
 export async function loader(args: LoaderFunctionArgs) {
   return handle(args, {
@@ -13,7 +14,21 @@ export async function action(args: ActionFunctionArgs) {
   return handle(args, {
     POST: {
       handler: (req) => proxy(req, { rawOutput: true }),
-      validator: ALLOW,
+      validator: validateProjectAccess(UserProjectPermission.CAN_MANAGE_PROJECT, {
+        getProjectId: (params) => params.id,
+      }),
+    },
+    DELETE: {
+      handler: proxy,
+      validator: validateProjectAccess(UserProjectPermission.CAN_MANAGE_PROJECT, {
+        getProjectId: (params) => params.id,
+      }),
+    },
+    PUT: {
+      handler: proxy,
+      validator: validateProjectAccess(UserProjectPermission.CAN_MANAGE_PROJECT, {
+        getProjectId: (params) => params.id,
+      }),
     },
   })
 }

utopia-remix/app/routes/v1.github.authentication.finish.tsx

@@ -1,11 +1,24 @@
 import { LoaderFunctionArgs } from '@remix-run/node'
-import { handle, handleOptions } from '../util/api.server'
+import { ensure, handle, handleOptions } from '../util/api.server'
 import { proxy } from '../util/proxy.server'
 import { ALLOW } from '../handlers/validators'
+import { Status } from '../util/statusCodes'
 
 export async function loader(args: LoaderFunctionArgs) {
   return handle(args, {
     OPTIONS: handleOptions,
-    GET: { handler: proxy, validator: ALLOW },
+    GET: { handler: handleFinish, validator: ALLOW },
+  })
+}
+
+async function handleFinish(req: Request) {
+  const proxyResponse = await proxy(req, { rawOutput: true })
+  ensure(proxyResponse instanceof Response, 'invalid response', Status.INTERNAL_ERROR)
+  ensure(proxyResponse.ok, proxyResponse.statusText, proxyResponse.status)
+
+  const body = await proxyResponse.text()
+  return new Response(body, {
+    headers: { 'content-type': 'text/html' },
+    status: 200,
   })
 }

utopia-remix/app/routes/v1.user.config.tsx

@@ -1,4 +1,4 @@
-import { LoaderFunctionArgs } from '@remix-run/node'
+import { ActionFunctionArgs, LoaderFunctionArgs } from '@remix-run/node'
 import { proxy } from '../util/proxy.server'
 import { handle, handleOptions } from '../util/api.server'
 import { ALLOW } from '../handlers/validators'
@@ -9,3 +9,9 @@ export async function loader(args: LoaderFunctionArgs) {
     GET: { handler: proxy, validator: ALLOW },
   })
 }
+
+export async function action(args: ActionFunctionArgs) {
+  return handle(args, {
+    POST: { handler: proxy, validator: ALLOW },
+  })
+}

utopia-remix/app/test-util.ts

@@ -109,7 +109,7 @@ export function newTestRequest(params?: {
   formData?: FormData
 }): Request {
   const path = (params?.path ?? '').replace(/^\/+/, '')
-  const req = new Request(`http://localhost:8002/` + path, {
+  const req = new Request(`http://localhost:8000/` + path, {
     method: params?.method,
     body: params?.formData,
   })

utopia-remix/app/util/api.server.ts

@@ -4,12 +4,12 @@ import * as cookie from 'cookie'
 import { UserDetails } from 'prisma-client'
 import { PrismaClientKnownRequestError } from 'prisma-client/runtime/library.js'
 import invariant from 'tiny-invariant'
-import { ServerEnvironment } from '../env.server'
+import { ALLOW } from '../handlers/validators'
 import { getUserFromSession } from '../models/session.server'
 import { ApiError } from './errors'
 import { Method } from './methods.server'
 import { Status } from './statusCodes'
-import { ALLOW } from '../handlers/validators'
+import { ServerEnvironment } from '../env.server'
 
 interface ErrorResponse {
   error: string
@@ -19,13 +19,15 @@ type EmptyResponse = Record<string, never>
 
 export type ApiResponse<T> = TypedResponse<T | ErrorResponse | EmptyResponse>
 
-const defaultResponseHeaders = new Headers({
-  'Access-Control-Allow-Origin': ServerEnvironment.CORSOrigin,
-  'Access-Control-Allow-Credentials': 'true',
-  'Access-Control-Allow-Headers': 'content-type, origin, cookie',
-  'Access-Control-Allow-Methods': 'GET,POST,PUT,DELETE,OPTIONS',
-  'Cache-control': 'no-cache',
-})
+function defaultResponseHeaders(): Headers {
+  return new Headers({
+    'Access-Control-Allow-Origin': ServerEnvironment.CORSOrigin,
+    'Access-Control-Allow-Credentials': 'true',
+    'Access-Control-Allow-Headers': 'content-type, origin, cookie',
+    'Access-Control-Allow-Methods': 'GET,POST,PUT,DELETE,OPTIONS',
+    'Cache-control': 'no-cache',
+  })
+}
 
 export function alwaysAllow(
   handler: (request: Request, params: Params<string>) => Promise<unknown>,
@@ -37,7 +39,7 @@ export function alwaysAllow(
 }
 
 async function optionsHandler(): Promise<TypedResponse<EmptyResponse>> {
-  return json({}, { headers: defaultResponseHeaders })
+  return json({}, { headers: defaultResponseHeaders() })
 }
 export const handleOptions = alwaysAllow(optionsHandler)
 
@@ -79,26 +81,27 @@ async function handleMethod<T>(
     if (validator != null) {
       await validator(request, params)
     }
+
     const resp = await fn(request, params)
     if (resp instanceof Response) {
-      const mergedHeaders = new Headers(defaultResponseHeaders)
+      let headers = new Headers()
       resp.headers.forEach((value, key) => {
-        mergedHeaders.set(key, value)
+        headers.set(key, value)
       })
       return new Response(resp.body, {
         status: resp.status,
-        headers: mergedHeaders,
+        headers: headers,
       })
     }
-    return json(resp, { headers: defaultResponseHeaders })
+    return json(resp, { headers: { ...defaultResponseHeaders() } })
   } catch (err) {
     const { message, status, name } = getErrorData(err)
 
     console.error(`${request.method} ${request.url}: ${message}`)
 
     return json(
       { error: name, status: status, message: message },
-      { headers: defaultResponseHeaders, status: status },
+      { headers: defaultResponseHeaders(), status: status },
     )
   }
 }
@@ -145,6 +148,7 @@ export async function proxiedResponse(response: Response): Promise<unknown> {
 }
 
 export const SESSION_COOKIE_NAME = 'JSESSIONID'
+
 function getSessionId(request: Request): string | null {
   const cookieHeader = request.headers.get('cookie') ?? ''
   const cookies = cookie.parse(cookieHeader)

utopia-remix/app/util/auth0.server.ts

@@ -1,13 +1,30 @@
+import urlJoin from 'url-join'
 import { ServerEnvironment } from '../env.server'
 
 export function auth0LoginURL(): string {
+  const behaviour = 'auto-close'
+
+  const useAuth0 =
+    ServerEnvironment.AUTH0_ENDPOINT !== '' &&
+    ServerEnvironment.AUTH0_CLIENT_ID !== '' &&
+    ServerEnvironment.AUTH0_REDIRECT_URI !== ''
+  if (!useAuth0) {
+    console.warn(
+      'Auth0 is disabled, if you need it be sure to set the AUTH0_ENDPOINT, AUTH0_CLIENT_ID, AUTH0_REDIRECT_URI environment variables',
+    )
+    const url = new URL(urlJoin(ServerEnvironment.BackendURL, 'authenticate'))
+    url.searchParams.set('code', 'logmein')
+    url.searchParams.set('onto', behaviour)
+    return url.href
+  }
+
   const url = new URL(`https://${ServerEnvironment.AUTH0_ENDPOINT}/authorize`)
   url.searchParams.set('scope', 'openid profile email')
   url.searchParams.set('response_type', 'code')
   url.searchParams.set('client_id', ServerEnvironment.AUTH0_CLIENT_ID)
 
   const redirectURL = new URL(ServerEnvironment.AUTH0_REDIRECT_URI)
-  redirectURL.searchParams.set('onto', 'auto-close')
+  redirectURL.searchParams.set('onto', behaviour)
 
   url.searchParams.set('redirect_uri', redirectURL.href)
   return url.href

utopia-remix/package.json

@@ -5,9 +5,9 @@
   "type": "module",
   "scripts": {
     "build": "remix build",
-    "dev": "remix dev --manual",
+    "dev": "remix dev -c \"node server.js\"",
     "lint": "eslint --ignore-path .gitignore --cache --cache-location ./node_modules/.cache/eslint .",
-    "start": "remix-serve ./build/index.js",
+    "start": "node ./server.js",
     "typecheck": "tsc",
     "format": "prettier --write .",
     "test": "./run-integration-tests.sh",
@@ -24,14 +24,17 @@
     "@radix-ui/react-tooltip": "1.0.7",
     "@radix-ui/themes": "2.0.3",
     "@remix-run/css-bundle": "2.5.1",
+    "@remix-run/express": "2.8.0",
     "@remix-run/node": "2.5.1",
     "@remix-run/react": "2.5.1",
     "@remix-run/serve": "2.5.1",
     "classnames": "2.5.1",
     "@tailwindcss/aspect-ratio": "0.4.2",
     "@tailwindcss/typography": "0.5.10",
     "cookie": "0.6.0",
+    "cors": "2.8.5",
     "dotenv": "16.4.1",
+    "express": "4.18.3",
     "framer-motion": "11.0.6",
     "isbot": "4",
     "moment": "2.30.1",
@@ -49,6 +52,8 @@
     "@babel/preset-env": "7.23.9",
     "@remix-run/dev": "2.5.1",
     "@types/cookie": "0.6.0",
+    "@types/cors": "2.8.17",
+    "@types/express": "4.17.21",
     "@types/jest": "29.5.12",
     "@types/node": "20.11.15",
     "@types/react": "18.2.20",
@@ -59,6 +64,7 @@
     "@vanilla-extract/dynamic": "2.1.0",
     "@vanilla-extract/recipes": "0.5.1",
     "@vanilla-extract/sprinkles": "1.6.1",
+    "chokidar": "3.6.0",
     "eslint": "8.38.0",
     "eslint-import-resolver-typescript": "3.6.1",
     "eslint-plugin-import": "2.28.1",
@@ -68,7 +74,8 @@
     "jest": "29.7.0",
     "prettier": "3.0.3",
     "prisma": "5.9.0",
-    "tailwindcss": "^3.4.1",
+    "source-map-support": "0.5.21",
+    "tailwindcss": "3.4.1",
     "ts-node": "10.9.2",
     "typescript": "5.1.6"
   },