feat: Add /execute endpoint and docker refactorings

Author: beneboy

envs/nodejs.yaml

@@ -1,7 +1,8 @@
 '@context': 'https://stenci.la/schema/v1/'
 type: Environment
 name: nodejs
-description: |
-  Contains only Node.js, the open-source, cross-platform JavaScript run-time environment.
+description: >
+  Contains only Node.js, the open-source, cross-platform JavaScript run-time
+  environment.
 adds:
   - nodejs

package.json

@@ -64,6 +64,7 @@
   "dependencies": {
     "await-spawn": "^2.1.2",
     "better-sqlite3": "^5.2.1",
+    "body-parser": "^1.18.3",
     "chalk": "^2.4.2",
     "del": "^3.0.0",
     "ellipsize": "^0.1.0",

src/Environment.ts

@@ -21,6 +21,9 @@ export enum Platform {
   UNIX, WIN, DOCKER
 }
 
+const DOCKER_DEFAULT_COMMAND = 'sh'
+const DOCKER_CONTAINER_ID_SHORT_LENGTH = 12
+
 /**
  * Parameters of a user session inside an environment
  */
@@ -49,6 +52,25 @@ export class SessionParameters {
    * Standard output stream
    */
   stdout: stream.Writable = process.stdout
+
+  /**
+   * CPU shares (only applies when using Docker platform). Priority of the container relative to other processes.
+   * The default is 1024, a higher number means higher priority for execution (when CPU contention exists).
+   */
+  cpuShares: Number = 1024
+
+  /**
+   * Memory limit (only applies when using Docker platform). Maximum amount of memory a container can use.
+   * Should be set in the format <number>[<unit>]). Number is a positive integer. Unit can be one of b, k, m, or g.
+   * Minimum is 4M.
+   */
+  memoryLimit: string = '0'
+
+  /**
+   * The ID of the container to attempt to use for an execution. It may be an empty string in which case the executor
+   * will start a new container.
+   */
+  containerId: string = ''
 }
 
 /**
@@ -350,11 +372,11 @@ export default class Environment {
   /**
    * Create variables for an environment.
    *
-   * This method is used in several other metho
+   * This method is used in several other methods
    * e.g. `within`, `enter`
    *
    * A 'pure' environment will only have available the executables that
-   * were exlicitly installed into the environment
+   * were explicitly installed into the environment
    *
    * @param pure Should the shell that this command is executed in be 'pure'?
    */
@@ -389,14 +411,42 @@ export default class Environment {
     })
   }
 
+  private async getDockerShellArgs (dockerCommand: string, sessionParameters: SessionParameters, daemonize: boolean = false): Promise<Array<string>> {
+    const { command, cpuShares, memoryLimit } = sessionParameters
+    const nixLocation = await nix.location(this.name)
+    const shellArgs = [
+          dockerCommand, '--interactive', '--tty', '--rm',
+          // Prepend the environment path to the PATH variable
+          '--env', `PATH=${nixLocation}/bin:${nixLocation}/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`,
+          // We also need to tell R where to find libraries
+          '--env', `R_LIBS_SITE=${nixLocation}/library`,
+          // Read-only bind mount of the Nix store
+          '--volume', '/nix/store:/nix/store:ro',
+          // Apply CPU shares
+          `--cpu-shares=${cpuShares}`,
+          // Apply memory limit
+          `--memory=${memoryLimit}`,
+          // We use Alpine Linux as a base image because it is very small but has some basic
+          // shell utilities (lkike ls and uname) that are good for debugging but also sometimes
+          // required for things like R
+          'alpine'
+        ].concat(
+          // Command to execute in the container
+          command ? command.split(' ') : DOCKER_DEFAULT_COMMAND
+        )
+
+    if (daemonize) shellArgs.splice(1, 0, '-d')
+
+    return shellArgs
+  }
+
   /**
    * Enter the a shell within the environment
    *
    * @param sessionParameters Parameters of the session
    */
   async enter (sessionParameters: SessionParameters) {
     let { command, platform, pure, stdin, stdout } = sessionParameters
-    const location = await nix.location(this.name)
 
     if (platform === undefined) {
       switch (os.platform()) {
@@ -417,22 +467,7 @@ export default class Environment {
         break
       case Platform.DOCKER:
         shellName = 'docker'
-        shellArgs = [
-          'run', '--interactive', '--tty', '--rm',
-          // Prepend the environment path to the PATH variable
-          '--env', `PATH=${location}/bin:${location}/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`,
-          // We also need to tell R where to find libraries
-          '--env', `R_LIBS_SITE=${location}/library`,
-          // Read-only bind mount of the Nix store
-          '--volume', '/nix/store:/nix/store:ro',
-          // We use Alpine Linux as a base image because it is very small but has some basic
-          // shell utilities (lkike ls and uname) that are good for debugging but also sometimes
-          // required for things like R
-          'alpine'
-        ].concat(
-          // Command to execute in the container
-          command ? command.split(' ') : 'sh'
-        )
+        shellArgs = await this.getDockerShellArgs('run', sessionParameters, false)
         break
       default:
         shellName = 'bash'
@@ -452,10 +487,14 @@ export default class Environment {
       // During development you'll need to use ---pure=false so that
       // node is available to run Nixster. In production, when a user
       // has installed a binary, this shouldn't be necessary
-      let nixsterPath = await spawn('which', ['nixster'])
-      const tempRcFile = tmp.fileSync()
-      fs.writeFileSync(tempRcFile.name, `alias nixster="${nixsterPath.toString().trim()}"\n`)
-      shellArgs.push('--rcfile', tempRcFile.name)
+      try {
+        let nixsterPath = await spawn('which', ['nixster'])
+        const tempRcFile = tmp.fileSync()
+        fs.writeFileSync(tempRcFile.name, `alias nixster="${nixsterPath.toString().trim()}"\n`)
+        shellArgs.push('--rcfile', tempRcFile.name)
+      } catch (e) {
+        // ignore
+      }
     }
 
     // Environment variables
@@ -512,4 +551,53 @@ export default class Environment {
 
     if (platform === Platform.UNIX && command) shellProcess.write(command + '\r')
   }
+
+  private async checkContainerRunning (containerId: string) {
+    const containerRegex = new RegExp(/^[^_\W]{12}$/)
+    if (containerRegex.exec(containerId) === null) {
+      throw new Error(`'${containerId}' is not a valid docker container ID.`)
+    }
+
+    // List running containers that match the containerId we are looking for. There should be only one or zero.
+    const dockerPsProcess = await spawn('docker', ['ps', '-q', '--filter', `id=${containerId}`])
+    const foundContainerId = dockerPsProcess.toString().trim()
+    return foundContainerId === containerId // foundContainerId should be either containerId or an empty string
+  }
+
+  /**
+   * Start a new Docker container and execute a command within it. The container daemonizes and keeps running
+   * (until the process it is running stops).
+   *
+   * Returns the short ID of the container that is running.
+   */
+  async execute (sessionParameters: SessionParameters): Promise<string> {
+    if (sessionParameters.platform !== Platform.DOCKER) {
+      throw new Error('Execute is only valid with the Docker platform.')
+    }
+
+    const shellArgs = await this.getDockerShellArgs('run', sessionParameters, true)
+
+    const dockerProcess = await spawn('docker', shellArgs)
+    return dockerProcess.toString().trim().substr(0, DOCKER_CONTAINER_ID_SHORT_LENGTH)
+  }
+
+  /**
+   * Build a Docker container for this environment
+   */
+  async dockerBuild () {
+    const requisites = await nix.requisites(this.name)
+    const dockerignore = `*\n${requisites.map(req => '!' + req).join('\n')}`
+    console.log(dockerignore)
+
+    // The Dockerfile does essentially the same as the `docker run` command
+    // generated above in `dockerRun`...
+    const location = await nix.location(this.name)
+    const dockerfile = `
+FROM alpine
+ENV PATH ${location}/bin:${location}/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ENV R_LIBS_SITE=${location}/library
+COPY /nix/store /nix/store
+    `
+    console.log(dockerfile)
+  }
 }

src/cli.ts

@@ -177,6 +177,12 @@ yargs
         alias: 'env',
         type: 'string'
       })
+      .option('docker', {
+        describe: 'Build the environment using Docker? Nix packages are stored on the host.',
+        type: 'boolean',
+        alias: 'd',
+        default: false
+      })
       .env('NIXSTER')
       .option('docker', {
         describe: 'Also build a Docker container for the environment?',

src/nix.ts

@@ -21,6 +21,7 @@ export function semver (version: string): string {
   let match = version.match(/^(\d+)(\.(\d+))?(\.(\d+))?(.*)?/)
   return match ? sprintf('%05i.%05i.%05i%s', match[1], match[3] || 0, match[5] || 0, match[6] || '') : version
 }
+
 // Register function in the database
 try {
   db.function('semver', semver)
@@ -218,7 +219,7 @@ export async function match (pkg: string): Promise<Array<any>> {
  * @param limit Limit on number of packages to return
  */
 export async function search (term: string, type: string = '', limit: number = 1000): Promise<Array<any>> {
-  term = term.replace("'", "\'")
+  term = term.replace('\'', '\'')
   // TODO: find a way to show the channel and description for the latest
   // version
   const stmt = db.prepare(`
@@ -246,19 +247,18 @@ export async function search (term: string, type: string = '', limit: number = 1
  * @param env The environment name
  */
 export async function built (env: string): Promise<boolean> {
-  return (await location(env)).length > 0
+  return (location(env)).length > 0
 }
 
 /**
  * Get the location of an environment within the Nix store
  *
  * @param env The environment name
  */
-export async function location (env: string): Promise<string> {
+export function location (env: string): string {
   const profile = path.join(profiles, env)
   if (!fs.existsSync(profile)) return ''
-  const readlink = await spawn('readlink', ['-f', profile])
-  return readlink
+  return fs.realpathSync(profile)
 }
 
 /**
@@ -267,8 +267,8 @@ export async function location (env: string): Promise<string> {
  * @param env The environment name
  * @param pkgs An array of normalized package names
  */
-export async function install (env: string, pkgs: Array<string>, clean: boolean = false) {
-  let channels: {[key: string]: any} = {}
+export async function install (env: string, pkgs: Array<string>, clean: boolean = false, inDocker: boolean = false) {
+  let channels: { [key: string]: any } = {}
   for (let pkg of pkgs) {
     let matches = await match(pkg)
     if (matches.length === 0) {
@@ -295,7 +295,23 @@ export async function install (env: string, pkgs: Array<string>, clean: boolean
     if (clean) args = args.concat('--remove-all')
     args = args.concat('--attr', channels[channel].map((pkg: any) => pkg.attr))
 
-    await spawn('nix-env', args, {
+    let command
+
+    if (inDocker) {
+      command = 'docker'
+      let dockerArgs = [
+          'run',
+          'stencila/nixster', 'nix-env'
+          // todo: write correct commands
+      ]
+      args = dockerArgs.concat(args)
+    } else {
+      command = 'nix-env'
+    }
+
+    console.log(args.join(' '))
+
+    await spawn(command, args, {
       stdio: 'inherit'
     })
   }
@@ -363,7 +379,7 @@ export async function requisites (env: string): Promise<Array<any>> {
   const query = await spawn('nix-store', [
     '--query',
     '--requisites',
-    await location(env)
+    location(env)
   ])
   const list = query.toString().trim()
   return list.length ? list.split('\n') : []

src/serve.ts

@@ -11,6 +11,11 @@ const expressWs = require('express-ws')(app)
 // Serve static assets from ./static
 app.use(express.static(path.join(__dirname, 'static')))
 
+// JSON Body Parsing
+const jsonParser = require('body-parser').json()
+app.use(jsonParser)
+
+// todo: rename shell to interact
 // Instantiate shell and set up data handlers
 expressWs.app.ws('/shell', async (ws: any, req: express.Request) => {
   try {
@@ -51,5 +56,24 @@ app.use((error: Error, req: express.Request, res: express.Response, next: any) =
   next(error)
 })
 
+expressWs.app.post('/execute', jsonParser, async (req: any, res: any) => {
+  // req: some JSON -> new SessionParameters object to start in env.execute (new execute method)
+  if (!req.body) return res.sendStatus(400)
+
+  const env = new Environment(req.body.environmentId)
+  const sessionParameters = new SessionParameters()
+  sessionParameters.platform = Platform.DOCKER
+  sessionParameters.command = req.body.command || ''
+
+  const containerId = await env.execute(sessionParameters)
+  res.status(200).json({
+    containerId: containerId
+  })
+})
+
+expressWs.app.post('/stop', async (req: any, res: any) => {
+  // req: some JSON -> with container ID that will stop the container
+})
+
 app.listen(3000)
 console.error('Listening on http://localhost:3000')