chore: Cleanup after e2e tests (#10748)

This PR should ensure state is cleaned up after e2e tests

---------

Co-authored-by: thunkar <gregojquiros@gmail.com>

yarn-project/end-to-end/src/fixtures/snapshot_manager.ts

@@ -17,6 +17,7 @@ import { deployInstance, registerContractClass } from '@aztec/aztec.js/deploymen
 import { type DeployL1ContractsArgs, createL1Clients, getL1ContractsConfigEnvVars, l1Artifacts } from '@aztec/ethereum';
 import { EthCheatCodesWithState, startAnvil } from '@aztec/ethereum/test';
 import { asyncMap } from '@aztec/foundation/async-map';
+import { randomBytes } from '@aztec/foundation/crypto';
 import { createLogger } from '@aztec/foundation/log';
 import { resolver, reviver } from '@aztec/foundation/serialize';
 import { TestDateProvider } from '@aztec/foundation/timer';
@@ -27,7 +28,9 @@ import { createAndStartTelemetryClient, getConfigEnvVars as getTelemetryConfig }
 import { type Anvil } from '@viem/anvil';
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
 import { copySync, removeSync } from 'fs-extra/esm';
-import { join } from 'path';
+import fs from 'fs/promises';
+import { tmpdir } from 'os';
+import path, { join } from 'path';
 import { type Hex, getContract } from 'viem';
 import { mnemonicToAccount } from 'viem/accounts';
 
@@ -50,6 +53,7 @@ export type SubsystemsContext = {
   watcher: AnvilTestWatcher;
   cheatCodes: CheatCodes;
   dateProvider: TestDateProvider;
+  directoryToCleanup?: string;
 };
 
 type SnapshotEntry = {
@@ -247,8 +251,12 @@ async function teardown(context: SubsystemsContext | undefined) {
     await context.proverNode?.stop();
     await context.aztecNode.stop();
     await context.acvmConfig?.cleanup();
+    await context.bbConfig?.cleanup();
     await context.anvil.stop();
     await context.watcher.stop();
+    if (context.directoryToCleanup) {
+      await fs.rm(context.directoryToCleanup, { recursive: true, force: true });
+    }
   } catch (err) {
     getLogger().error('Error during teardown', err);
   }
@@ -273,7 +281,15 @@ async function setupFromFresh(
   // Fetch the AztecNode config.
   // TODO: For some reason this is currently the union of a bunch of subsystems. That needs fixing.
   const aztecNodeConfig: AztecNodeConfig & SetupOptions = { ...getConfigEnvVars(), ...opts };
-  aztecNodeConfig.dataDirectory = statePath;
+
+  // Create a temp directory for all ephemeral state and cleanup afterwards
+  const directoryToCleanup = path.join(tmpdir(), randomBytes(8).toString('hex'));
+  await fs.mkdir(directoryToCleanup, { recursive: true });
+  if (statePath === undefined) {
+    aztecNodeConfig.dataDirectory = directoryToCleanup;
+  } else {
+    aztecNodeConfig.dataDirectory = statePath;
+  }
 
   // Start anvil. We go via a wrapper script to ensure if the parent dies, anvil dies.
   logger.verbose('Starting anvil...');
@@ -363,12 +379,13 @@ async function setupFromFresh(
       `0x${proverNodePrivateKey!.toString('hex')}`,
       aztecNodeConfig,
       aztecNode,
+      path.join(directoryToCleanup, randomBytes(8).toString('hex')),
     );
   }
 
   logger.verbose('Creating pxe...');
   const pxeConfig = getPXEServiceConfig();
-  pxeConfig.dataDirectory = statePath;
+  pxeConfig.dataDirectory = statePath ?? path.join(directoryToCleanup, randomBytes(8).toString('hex'));
   const pxe = await createPXEService(aztecNode, pxeConfig);
 
   const cheatCodes = await CheatCodes.create(aztecNodeConfig.l1RpcUrl, pxe);
@@ -389,6 +406,7 @@ async function setupFromFresh(
     watcher,
     cheatCodes,
     dateProvider,
+    directoryToCleanup,
   };
 }
 
@@ -398,6 +416,9 @@ async function setupFromFresh(
 async function setupFromState(statePath: string, logger: Logger): Promise<SubsystemsContext> {
   logger.verbose(`Initializing with saved state at ${statePath}...`);
 
+  const directoryToCleanup = path.join(tmpdir(), randomBytes(8).toString('hex'));
+  await fs.mkdir(directoryToCleanup, { recursive: true });
+
   // TODO: For some reason this is currently the union of a bunch of subsystems. That needs fixing.
   const aztecNodeConfig: AztecNodeConfig & SetupOptions = JSON.parse(
     readFileSync(`${statePath}/aztec_node_config.json`, 'utf-8'),
@@ -446,7 +467,12 @@ async function setupFromState(statePath: string, logger: Logger): Promise<Subsys
     logger.verbose('Creating and syncing a simulated prover node...');
     const proverNodePrivateKey = getPrivateKeyFromIndex(2);
     const proverNodePrivateKeyHex: Hex = `0x${proverNodePrivateKey!.toString('hex')}`;
-    proverNode = await createAndSyncProverNode(proverNodePrivateKeyHex, aztecNodeConfig, aztecNode);
+    proverNode = await createAndSyncProverNode(
+      proverNodePrivateKeyHex,
+      aztecNodeConfig,
+      aztecNode,
+      path.join(directoryToCleanup, randomBytes(8).toString('hex')),
+    );
   }
 
   logger.verbose('Creating pxe...');
@@ -472,6 +498,7 @@ async function setupFromState(statePath: string, logger: Logger): Promise<Subsys
     watcher,
     cheatCodes,
     dateProvider,
+    directoryToCleanup,
   };
 }
 

yarn-project/end-to-end/src/fixtures/utils.ts

@@ -36,6 +36,7 @@ import {
   l1Artifacts,
 } from '@aztec/ethereum';
 import { EthCheatCodesWithState, startAnvil } from '@aztec/ethereum/test';
+import { randomBytes } from '@aztec/foundation/crypto';
 import { retryUntil } from '@aztec/foundation/retry';
 import { TestDateProvider } from '@aztec/foundation/timer';
 import { FeeJuiceContract } from '@aztec/noir-contracts.js/FeeJuice';
@@ -48,6 +49,8 @@ import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
 import { createAndStartTelemetryClient, getConfigEnvVars as getTelemetryConfig } from '@aztec/telemetry-client/start';
 
 import { type Anvil } from '@viem/anvil';
+import fs from 'fs/promises';
+import { tmpdir } from 'os';
 import * as path from 'path';
 import {
   type Account,
@@ -144,9 +147,20 @@ export async function setupPXEService(
   teardown: () => Promise<void>;
 }> {
   const pxeServiceConfig = { ...getPXEServiceConfig(), ...opts };
+
+  // If no data directory provided, create a temp directory and clean up afterwards
+  const configuredDataDirectory = pxeServiceConfig.dataDirectory;
+  if (!configuredDataDirectory) {
+    pxeServiceConfig.dataDirectory = path.join(tmpdir(), randomBytes(8).toString('hex'));
+  }
+
   const pxe = await createPXEService(aztecNode, pxeServiceConfig, useLogSuffix, proofCreator);
 
-  const teardown = async () => {};
+  const teardown = async () => {
+    if (!configuredDataDirectory) {
+      await fs.rm(pxeServiceConfig.dataDirectory!, { recursive: true, force: true });
+    }
+  };
 
   return {
     pxe,
@@ -303,6 +317,13 @@ export async function setup(
   const config = { ...getConfigEnvVars(), ...opts };
   const logger = getLogger();
 
+  // Create a temp directory for any services that need it and cleanup later
+  const directoryToCleanup = path.join(tmpdir(), randomBytes(8).toString('hex'));
+  await fs.mkdir(directoryToCleanup, { recursive: true });
+  if (!config.dataDirectory) {
+    config.dataDirectory = directoryToCleanup;
+  }
+
   let anvil: Anvil | undefined;
 
   if (!config.l1RpcUrl) {
@@ -427,11 +448,16 @@ export async function setup(
     logger.verbose('Creating and syncing a simulated prover node...');
     const proverNodePrivateKey = getPrivateKeyFromIndex(2);
     const proverNodePrivateKeyHex: Hex = `0x${proverNodePrivateKey!.toString('hex')}`;
-    proverNode = await createAndSyncProverNode(proverNodePrivateKeyHex, config, aztecNode);
+    proverNode = await createAndSyncProverNode(
+      proverNodePrivateKeyHex,
+      config,
+      aztecNode,
+      path.join(directoryToCleanup, randomBytes(8).toString('hex')),
+    );
   }
 
   logger.verbose('Creating a pxe...');
-  const { pxe } = await setupPXEService(aztecNode!, pxeOpts, logger);
+  const { pxe, teardown: pxeTeardown } = await setupPXEService(aztecNode!, pxeOpts, logger);
 
   if (!config.skipProtocolContracts) {
     logger.verbose('Setting up Fee Juice...');
@@ -444,6 +470,8 @@ export async function setup(
   const cheatCodes = await CheatCodes.create(config.l1RpcUrl, pxe!);
 
   const teardown = async () => {
+    await pxeTeardown();
+
     if (aztecNode instanceof AztecNodeService) {
       await aztecNode?.stop();
     }
@@ -454,8 +482,19 @@ export async function setup(
       await acvmConfig.cleanup();
     }
 
+    if (bbConfig?.cleanup) {
+      // remove the temp directory created for the acvm
+      logger.verbose(`Cleaning up BB state`);
+      await bbConfig.cleanup();
+    }
+
     await anvil?.stop();
     await watcher.stop();
+
+    if (directoryToCleanup) {
+      logger.verbose(`Cleaning up data directory at ${directoryToCleanup}`);
+      await fs.rm(directoryToCleanup, { recursive: true, force: true });
+    }
   };
 
   return {
@@ -659,6 +698,7 @@ export async function createAndSyncProverNode(
   proverNodePrivateKey: `0x${string}`,
   aztecNodeConfig: AztecNodeConfig,
   aztecNode: AztecNode,
+  dataDirectory: string,
 ) {
   // Disable stopping the aztec node as the prover coordination test will kill it otherwise
   // This is only required when stopping the prover node for testing
@@ -669,7 +709,7 @@ export async function createAndSyncProverNode(
   };
 
   // Creating temp store and archiver for simulated prover node
-  const archiverConfig = { ...aztecNodeConfig, dataDirectory: undefined };
+  const archiverConfig = { ...aztecNodeConfig, dataDirectory };
   const archiver = await createArchiver(archiverConfig, new NoopTelemetryClient(), { blockUntilSync: true });
 
   // Prover node config is for simulated proofs

yarn-project/kv-store/src/indexeddb/store.ts

@@ -1,6 +1,6 @@
 import { type Logger } from '@aztec/foundation/log';
 
-import { type DBSchema, type IDBPDatabase, openDB } from 'idb';
+import { type DBSchema, type IDBPDatabase, deleteDB, openDB } from 'idb';
 
 import { type AztecAsyncArray } from '../interfaces/array.js';
 import { type Key } from '../interfaces/common.js';
@@ -183,7 +183,8 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
   /** Deletes this store and removes the database */
   delete() {
     this.#containers.clear();
-    return Promise.resolve(this.#rootDB.deleteObjectStore('data'));
+    this.#rootDB.close();
+    return deleteDB(this.#name);
   }
 
   estimateSize(): { mappingSize: number; actualSize: number; numItems: number } {

yarn-project/kv-store/src/interfaces/array_test_suite.ts

@@ -20,6 +20,10 @@ export function describeAztecArray(
       arr = store.openArray<number>('test');
     });
 
+    afterEach(async () => {
+      await store.delete();
+    });
+
     async function length(sut: AztecAsyncArray<number> | AztecArray<number> = arr) {
       return isSyncStore(store) && !forceAsync
         ? (sut as AztecArray<number>).length

yarn-project/kv-store/src/interfaces/map_test_suite.ts

@@ -21,6 +21,10 @@ export function describeAztecMap(
       map = store.openMultiMap<string | [number, string], string>('test');
     });
 
+    afterEach(async () => {
+      await store.delete();
+    });
+
     async function get(key: Key, sut: AztecAsyncMap<any, any> | AztecMap<any, any> = map) {
       return isSyncStore(store) && !forceAsync
         ? (sut as AztecMultiMap<any, any>).get(key)

yarn-project/kv-store/src/interfaces/set_test_suite.ts

@@ -21,6 +21,10 @@ export function describeAztecSet(
       set = store.openSet<string>('test');
     });
 
+    afterEach(async () => {
+      await store.delete();
+    });
+
     async function has(key: string) {
       return isSyncStore(store) && !forceAsync
         ? (set as AztecSet<string>).has(key)

yarn-project/kv-store/src/interfaces/singleton_test_suite.ts

@@ -18,6 +18,10 @@ export function describeAztecSingleton(
       singleton = store.openSingleton<string>('test');
     });
 
+    afterEach(async () => {
+      await store.delete();
+    });
+
     async function get() {
       return isSyncStore(store) && !forceAsync
         ? (singleton as AztecSingleton<string>).get()

yarn-project/kv-store/src/interfaces/store_test_suite.ts

@@ -32,21 +32,25 @@ export function describeAztecStore(
       expect(await get(forkedStore, forkedSingleton)).to.equal('bar');
       await forkedSingleton.delete();
       expect(await get(store, singleton)).to.equal('foo');
+      await forkedStore.delete();
     };
 
     it('forks a persistent store', async () => {
       const store = await getPersistentStore();
       await itForks(store);
+      await store.delete();
     });
 
     it('forks a persistent store with no path', async () => {
       const store = await getPersistentNoPathStore();
       await itForks(store);
+      await store.delete();
     });
 
     it('forks an ephemeral store', async () => {
       const store = await getEphemeralStore();
       await itForks(store);
+      await store.delete();
     });
   });
 }

yarn-project/kv-store/src/lmdb/counter.test.ts

@@ -3,18 +3,30 @@ import { toArray } from '@aztec/foundation/iterable';
 
 import { expect, use } from 'chai';
 import chaiAsPromised from 'chai-as-promised';
+import fs from 'fs/promises';
 import { type Database, open } from 'lmdb';
 import forEach from 'mocha-each';
+import { tmpdir } from 'os';
+import path from 'path';
 
 import { LmdbAztecCounter } from './counter.js';
 
 use(chaiAsPromised);
 
 describe('LmdbAztecCounter', () => {
   let db: Database;
+  let dir: string;
 
-  beforeEach(() => {
-    db = open({} as any);
+  beforeEach(async () => {
+    dir = path.join(tmpdir(), randomBytes(8).toString('hex'));
+    await fs.mkdir(dir, { recursive: true });
+    db = open({ path: dir } as any);
+  });
+
+  afterEach(async () => {
+    await db.drop();
+    await db.close();
+    await fs.rm(dir, { recursive: true, force: true });
   });
 
   forEach([