feat(setup): non interactive setup with artifacts download

Author: ctrlc03

packages/actions/package.json

@@ -42,6 +42,7 @@
     "dependencies": {
         "@adobe/node-fetch-retry": "^2.2.0",
         "@aws-sdk/client-ec2": "^3.358.0",
+        "@aws-sdk/client-s3": "^3.329.0",
         "@aws-sdk/client-ssm": "^3.358.0",
         "@octokit/auth-oauth-device": "^4.0.4",
         "@types/winston": "^2.4.4",

packages/actions/src/helpers/utils.ts

@@ -2,6 +2,7 @@ import { Firestore } from "firebase/firestore"
 import fs, { ReadPosition } from "fs"
 import { utils as ffUtils } from "ffjavascript"
 import winston, { Logger } from "winston"
+import { S3Client, GetObjectCommand, HeadObjectCommand } from "@aws-sdk/client-s3"
 import { 
     CircuitMetadata, 
     Contribution, 
@@ -27,14 +28,17 @@ import {
     getWasmStorageFilePath, 
     getZkeyStorageFilePath
 } from "./storage"
+import { blake512FromPath } from "./crypto"
+import { Readable } from "stream"
 
 /**
  * Parse and validate that the ceremony configuration is correct
  * @notice this does not upload any files to storage
  * @param path <string> - the path to the configuration file
+ * @param cleanup <boolean> - whether to delete the r1cs file after parsing
  * @returns any - the data to pass to the cloud function for setup and the circuit artifacts
  */
-export const parseCeremonyFile = (path: string): SetupCeremonyData => {
+export const parseCeremonyFile = async (path: string, cleanup: boolean = false): Promise<SetupCeremonyData> => {
     // check that the path exists
     if (!fs.existsSync(path)) throw new Error("The provided path to the configuration file does not exist. Please provide an absolute path and try again.")
 
@@ -82,15 +86,48 @@ export const parseCeremonyFile = (path: string): SetupCeremonyData => {
             circuitArtifacts.push({
                 artifacts: artifacts
             })
-            const r1csPath = artifacts.r1csLocalFilePath
-            const wasmPath = artifacts.wasmLocalFilePath
+            const r1csPath = artifacts.r1csStoragePath
+            const wasmPath = artifacts.wasmStoragePath
+
+            // where we storing the r1cs downloaded
+            const localR1csPath = `./${circuitData.name}.r1cs`
+
+            // check that the artifacts exist in S3
+            const s3 = new S3Client({region: 'us-east-1'})
+
+            try {
+                await s3.send(new HeadObjectCommand({
+                    Bucket: circuitData.artifacts.bucket,
+                    Key: r1csPath 
+                }))
+            } catch (error: any) {
+                throw new Error("The r1cs file seems to not exist. Please ensure this is correct and that the object is publicly available.")
+            }
+            
+            try {
+                await s3.send(new HeadObjectCommand({
+                    Bucket: artifacts.bucket,
+                    Key: wasmPath 
+                }))
+            } catch (error: any) {
+                throw new Error("The wasm file seems to not exist. Please ensure this is correct and that the object is publicly available.")
+            }
 
-            // ensure that the artifact exist locally
-            if (!fs.existsSync(r1csPath)) throw new Error("The path to the r1cs file does not exist. Please ensure this is correct and that an absolute path is provided.")
-            if (!fs.existsSync(wasmPath)) throw new Error("The path to the wasm file does not exist. Please ensure this is correct and that an absolute path is provided.")
+            // download the r1cs to extract the metadata
+            const command = new GetObjectCommand({ Bucket: artifacts.bucket, Key: artifacts.r1csStoragePath })
+            const response = await s3.send(command)
+
+            const fileStream = fs.createWriteStream(localR1csPath)
+            if (response.$metadata.httpStatusCode !== 200) {
+                throw new Error("There was an error while trying to download the r1cs file. Please check that the file has the correct permissions (public) set.")
+            }
+            // const streamPipeline = promisify(pipeline)
+            if (response.Body instanceof Readable) {
+                response.Body.pipe(fileStream)
+            } 
 
             // extract the metadata from the r1cs
-            const metadata = getR1CSInfo(r1csPath)
+            const metadata = getR1CSInfo(localR1csPath)
 
             // validate that the circuit hash and template links are valid
             const template = circuitData.template
@@ -101,6 +138,9 @@ export const parseCeremonyFile = (path: string): SetupCeremonyData => {
             const hashMatch = template.commitHash.match(commitHashPattern)
             if (!hashMatch || hashMatch.length === 0 || hashMatch.length > 1) throw new Error("You should provide a valid commit hash of the circuit templates.")
 
+            // calculate the hash of the r1cs file
+            const r1csBlake2bHash = await blake512FromPath(localR1csPath)
+
             const circuitPrefix = extractPrefix(circuitData.name)
 
             // filenames
@@ -124,7 +164,8 @@ export const parseCeremonyFile = (path: string): SetupCeremonyData => {
                 potStoragePath: potStorageFilePath,
                 r1csStoragePath: r1csStorageFilePath,
                 wasmStoragePath: wasmStorageFilePath,
-                initialZkeyStoragePath: zkeyStorageFilePath
+                initialZkeyStoragePath: zkeyStorageFilePath,
+                r1csBlake2bHash: r1csBlake2bHash
             }
 
             // validate that the compiler hash is a valid hash 
@@ -177,6 +218,9 @@ export const parseCeremonyFile = (path: string): SetupCeremonyData => {
             }
 
             circuits.push(circuit)
+
+            // remove the local r1cs download (if used for verifying the config only vs setup)
+            if (cleanup) fs.unlinkSync(localR1csPath)
         }
 
         const setupData: SetupCeremonyData = {

packages/actions/src/types/index.ts

@@ -623,8 +623,9 @@ export type SetupCeremonyData = {
 
 export type CeremonySetupTemplateCircuitArtifacts = {
     artifacts: {
-        r1csLocalFilePath: string
-        wasmLocalFilePath: string
+        bucket: string
+        r1csStoragePath: string
+        wasmStoragePath: string
     }
 }
 

packages/actions/test/data/artifacts/ceremonySetup.json

@@ -1,13 +1,13 @@
 {
-    "title": "Example Ceremony",
+    "title": "Test dev ceremony",
     "description": "This is an example ceremony",
-    "startDate": "2023-07-07T00:00:00",
-    "endDate": "2023-07-10T00:00:00",
+    "startDate": "2023-08-07T00:00:00",
+    "endDate": "2023-09-10T00:00:00",
     "timeoutMechanismType": "FIXED",
     "penalty": 10,
     "circuits": [
         {
-            "description": "description",
+            "description": "description1",
             "compiler": {
                 "version": "1.0",
                 "commitHash": "b7ad01b11f9b4195e38ecc772291251260ab2c67"
@@ -21,8 +21,9 @@
                 "cfOrVm": "CF"
             },
             "artifacts": {
-                "r1csLocalFilePath": "./circuit.r1cs",
-                "wasmLocalFilePath": "./circuit.wasm"
+                "bucket": "test-qfi-p0tion-development-environment",
+                "r1csStoragePath": "circuits/circuit/circuit.r1cs", 
+                "wasmStoragePath": "circuits/circuit/circuit.wasm"  
             },
             "name": "circuit",
             "dynamicThreshold": 0,

packages/backend/src/lib/utils.ts

@@ -300,7 +300,7 @@ export const deleteObject = async (bucketName: string, objectKey: string) => {
 
     // Prepare command.
     const command = new DeleteObjectCommand({ Bucket: bucketName, Key: objectKey })
-
+    
     // Execute command.
     const data = await client.send(command)
 

packages/phase2cli/src/commands/setup.ts

@@ -2,11 +2,13 @@
 
 import { zKey } from "snarkjs"
 import boxen from "boxen"
-import { createWriteStream, Dirent, renameSync } from "fs"
+import { createWriteStream, Dirent, renameSync, createReadStream } from "fs"
 import { pipeline } from "node:stream"
 import { promisify } from "node:util"
 import fetch from "node-fetch"
 import { Functions } from "firebase/functions"
+import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3"
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner"
 import {
     CeremonyTimeoutType,
     CircomCompilerData,
@@ -62,6 +64,7 @@ import {
     getFileStats,
     checkAndMakeNewDirectoryIfNonexistent
 } from "../lib/files.js"
+import { Readable } from "stream"
 
 /**
  * Handle whatever is needed to obtain the input data for a circuit that the coordinator would like to add to the ceremony.
@@ -501,38 +504,54 @@ const setup = async (cmd: { template?: string, auth?: string}) => {
     // if there is the file option, then set up the non interactively
     if (cmd.template) {
         // 1. parse the file
-        // tmp data
-        const setupCeremonyData = parseCeremonyFile(cmd.template!)
+        // tmp data - do not cleanup files as we need them 
+        const setupCeremonyData = await parseCeremonyFile(cmd.template!)
         // final setup data
         const ceremonySetupData = setupCeremonyData
 
         // create a new bucket
         const bucketName = await handleCeremonyBucketCreation(firebaseFunctions, ceremonySetupData.ceremonyPrefix)
         console.log(`\n${theme.symbols.success} Ceremony bucket name: ${theme.text.bold(bucketName)}`)
 
+        // create S3 clienbt
+        const s3 = new S3Client({region: 'us-east-1'})
+
         // loop through each circuit
-        for (const circuit of setupCeremonyData.circuits) {
+        for await (const circuit of setupCeremonyData.circuits) {
             // Local paths.
             const index = ceremonySetupData.circuits.indexOf(circuit)
-            const r1csLocalPathAndFileName = setupCeremonyData.circuitArtifacts[index].artifacts.r1csLocalFilePath
-            const wasmLocalPathAndFileName = setupCeremonyData.circuitArtifacts[index].artifacts.wasmLocalFilePath
+            const r1csLocalPathAndFileName = `./${circuit.name}.r1cs`
+            const wasmLocalPathAndFileName = `./${circuit.name}.wasm`
             const potLocalPathAndFileName = getPotLocalFilePath(circuit.files.potFilename)
             const zkeyLocalPathAndFileName = getZkeyLocalFilePath(circuit.files.initialZkeyFilename)
 
-            // 2. download the pot
+            // 2. download the pot and wasm files
             const streamPipeline = promisify(pipeline)
-            const response = await fetch(`${potFileDownloadMainUrl}${circuit.files.potFilename}`)
+            const potResponse = await fetch(`${potFileDownloadMainUrl}${circuit.files.potFilename}`)
 
             // Handle errors.
-            if (!response.ok && response.status !== 200) showError("Error while setting up the ceremony. Could not download the powers of tau file.", true)
+            if (!potResponse.ok && potResponse.status !== 200) showError("Error while setting up the ceremony. Could not download the powers of tau file.", true)
 
-            await streamPipeline(response.body!, createWriteStream(potLocalPathAndFileName))
+            await streamPipeline(potResponse.body!, createWriteStream(potLocalPathAndFileName))
+
+            // download the wasm to calculate the hash
+            const command = new GetObjectCommand({ Bucket: ceremonySetupData.circuitArtifacts[index].artifacts.bucket, Key: ceremonySetupData.circuitArtifacts[index].artifacts.wasmStoragePath })
+
+            const response = await s3.send(command)
+
+            const fileStream = createWriteStream(wasmLocalPathAndFileName)
+            if (response.$metadata.httpStatusCode !== 200) {
+                throw new Error("There was an error while trying to download the wasm file. Please check that the file has the correct permissions (public) set.")
+            }
+            // const streamPipeline = promisify(pipeline)
+            if (response.Body instanceof Readable) {
+                response.Body.pipe(fileStream)
+            } 
 
             // 3. generate the zKey
             await zKey.newZKey(r1csLocalPathAndFileName, potLocalPathAndFileName, zkeyLocalPathAndFileName, undefined)
 
             // 4. calculate the hashes
-            const r1csBlake2bHash = await blake512FromPath(r1csLocalPathAndFileName)
             const wasmBlake2bHash = await blake512FromPath(wasmLocalPathAndFileName)
             const potBlake2bHash = await blake512FromPath(potLocalPathAndFileName)
             const initialZkeyBlake2bHash = await blake512FromPath(zkeyLocalPathAndFileName)
@@ -557,7 +576,7 @@ const setup = async (cmd: { template?: string, auth?: string}) => {
                 circuit.files.potFilename
             )
 
-            // Upload R1CS to Storage.
+            // Move r1cs between buckets
             await handleCircuitArtifactUploadToStorage(
                 firebaseFunctions,
                 bucketName,
@@ -566,7 +585,7 @@ const setup = async (cmd: { template?: string, auth?: string}) => {
                 circuit.files.r1csFilename
             )
 
-            // Upload WASM to Storage.
+            // Move wasm between buckets.
             await handleCircuitArtifactUploadToStorage(
                 firebaseFunctions,
                 bucketName,
@@ -575,11 +594,13 @@ const setup = async (cmd: { template?: string, auth?: string}) => {
                 circuit.files.wasmFilename
             )
 
+            // @todo move the artifacts from the origin bucket to the new bucket 
+            // instead of uploading r1cs and wasm
+
             // 6 update the setup data object
             ceremonySetupData.circuits[index].files = {
                 ...circuit.files,
                 potBlake2bHash: potBlake2bHash,
-                r1csBlake2bHash: r1csBlake2bHash,
                 wasmBlake2bHash: wasmBlake2bHash,
                 initialZkeyBlake2bHash: initialZkeyBlake2bHash
             }

packages/phase2cli/src/commands/validate.ts

@@ -6,11 +6,11 @@ import { showError } from "../lib/errors.js"
  */
 const validate = async (cmd: { template: string, constraints?: number }) => {
     try {
-        // parse the file
-        const parsedFile = parseCeremonyFile(cmd.template)
+        // parse the file and cleanup after
+        const parsedFile = await parseCeremonyFile(cmd.template, true)
         // check whether we have a constraints option otherwise default to 1M
         const constraints = cmd.constraints || 1000000
-        for (const circuit of parsedFile.circuits) {
+        for await (const circuit of parsedFile.circuits) {
             if (circuit.metadata.constraints > constraints) {
                 console.log(false)
                 process.exit(0)

yarn.lock

@@ -6146,6 +6146,7 @@ __metadata:
   dependencies:
     "@adobe/node-fetch-retry": ^2.2.0
     "@aws-sdk/client-ec2": ^3.358.0
+    "@aws-sdk/client-s3": ^3.329.0
     "@aws-sdk/client-ssm": ^3.358.0
     "@ethersproject/providers": ^5.7.2
     "@nomicfoundation/hardhat-chai-matchers": ^1.0.6