From 284b0a496f42813b956e55fbcd41c864dd278241 Mon Sep 17 00:00:00 2001 From: PhilWindle <60546371+PhilWindle@users.noreply.github.com> Date: Tue, 17 Dec 2024 10:51:00 +0000 Subject: [PATCH] chore: Cleanup after e2e tests (#10748) This PR should ensure state is cleaned up after e2e tests --------- Co-authored-by: thunkar --- .../src/fixtures/snapshot_manager.ts | 35 ++++++++++++-- yarn-project/end-to-end/src/fixtures/utils.ts | 48 +++++++++++++++++-- yarn-project/kv-store/src/indexeddb/store.ts | 5 +- .../src/interfaces/array_test_suite.ts | 4 ++ .../kv-store/src/interfaces/map_test_suite.ts | 4 ++ .../kv-store/src/interfaces/set_test_suite.ts | 4 ++ .../src/interfaces/singleton_test_suite.ts | 4 ++ .../src/interfaces/store_test_suite.ts | 4 ++ .../kv-store/src/lmdb/counter.test.ts | 16 ++++++- yarn-project/kv-store/src/lmdb/store.ts | 19 ++++---- .../kv-store/src/stores/l2_tips_store.test.ts | 4 ++ 11 files changed, 125 insertions(+), 22 deletions(-) diff --git a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts index 58252091655..2acb78f7d47 100644 --- a/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts +++ b/yarn-project/end-to-end/src/fixtures/snapshot_manager.ts @@ -17,6 +17,7 @@ import { deployInstance, registerContractClass } from '@aztec/aztec.js/deploymen import { type DeployL1ContractsArgs, createL1Clients, getL1ContractsConfigEnvVars, l1Artifacts } from '@aztec/ethereum'; import { EthCheatCodesWithState, startAnvil } from '@aztec/ethereum/test'; import { asyncMap } from '@aztec/foundation/async-map'; +import { randomBytes } from '@aztec/foundation/crypto'; import { createLogger } from '@aztec/foundation/log'; import { resolver, reviver } from '@aztec/foundation/serialize'; import { TestDateProvider } from '@aztec/foundation/timer'; @@ -27,7 +28,9 @@ import { createAndStartTelemetryClient, getConfigEnvVars as getTelemetryConfig } import { type Anvil } from '@viem/anvil'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; import { copySync, removeSync } from 'fs-extra/esm'; -import { join } from 'path'; +import fs from 'fs/promises'; +import { tmpdir } from 'os'; +import path, { join } from 'path'; import { type Hex, getContract } from 'viem'; import { mnemonicToAccount } from 'viem/accounts'; @@ -50,6 +53,7 @@ export type SubsystemsContext = { watcher: AnvilTestWatcher; cheatCodes: CheatCodes; dateProvider: TestDateProvider; + directoryToCleanup?: string; }; type SnapshotEntry = { @@ -247,8 +251,12 @@ async function teardown(context: SubsystemsContext | undefined) { await context.proverNode?.stop(); await context.aztecNode.stop(); await context.acvmConfig?.cleanup(); + await context.bbConfig?.cleanup(); await context.anvil.stop(); await context.watcher.stop(); + if (context.directoryToCleanup) { + await fs.rm(context.directoryToCleanup, { recursive: true, force: true }); + } } catch (err) { getLogger().error('Error during teardown', err); } @@ -273,7 +281,15 @@ async function setupFromFresh( // Fetch the AztecNode config. // TODO: For some reason this is currently the union of a bunch of subsystems. That needs fixing. const aztecNodeConfig: AztecNodeConfig & SetupOptions = { ...getConfigEnvVars(), ...opts }; - aztecNodeConfig.dataDirectory = statePath; + + // Create a temp directory for all ephemeral state and cleanup afterwards + const directoryToCleanup = path.join(tmpdir(), randomBytes(8).toString('hex')); + await fs.mkdir(directoryToCleanup, { recursive: true }); + if (statePath === undefined) { + aztecNodeConfig.dataDirectory = directoryToCleanup; + } else { + aztecNodeConfig.dataDirectory = statePath; + } // Start anvil. We go via a wrapper script to ensure if the parent dies, anvil dies. logger.verbose('Starting anvil...'); @@ -363,12 +379,13 @@ async function setupFromFresh( `0x${proverNodePrivateKey!.toString('hex')}`, aztecNodeConfig, aztecNode, + path.join(directoryToCleanup, randomBytes(8).toString('hex')), ); } logger.verbose('Creating pxe...'); const pxeConfig = getPXEServiceConfig(); - pxeConfig.dataDirectory = statePath; + pxeConfig.dataDirectory = statePath ?? path.join(directoryToCleanup, randomBytes(8).toString('hex')); const pxe = await createPXEService(aztecNode, pxeConfig); const cheatCodes = await CheatCodes.create(aztecNodeConfig.l1RpcUrl, pxe); @@ -389,6 +406,7 @@ async function setupFromFresh( watcher, cheatCodes, dateProvider, + directoryToCleanup, }; } @@ -398,6 +416,9 @@ async function setupFromFresh( async function setupFromState(statePath: string, logger: Logger): Promise { logger.verbose(`Initializing with saved state at ${statePath}...`); + const directoryToCleanup = path.join(tmpdir(), randomBytes(8).toString('hex')); + await fs.mkdir(directoryToCleanup, { recursive: true }); + // TODO: For some reason this is currently the union of a bunch of subsystems. That needs fixing. const aztecNodeConfig: AztecNodeConfig & SetupOptions = JSON.parse( readFileSync(`${statePath}/aztec_node_config.json`, 'utf-8'), @@ -446,7 +467,12 @@ async function setupFromState(statePath: string, logger: Logger): Promise Promise; }> { const pxeServiceConfig = { ...getPXEServiceConfig(), ...opts }; + + // If no data directory provided, create a temp directory and clean up afterwards + const configuredDataDirectory = pxeServiceConfig.dataDirectory; + if (!configuredDataDirectory) { + pxeServiceConfig.dataDirectory = path.join(tmpdir(), randomBytes(8).toString('hex')); + } + const pxe = await createPXEService(aztecNode, pxeServiceConfig, useLogSuffix, proofCreator); - const teardown = async () => {}; + const teardown = async () => { + if (!configuredDataDirectory) { + await fs.rm(pxeServiceConfig.dataDirectory!, { recursive: true, force: true }); + } + }; return { pxe, @@ -303,6 +317,13 @@ export async function setup( const config = { ...getConfigEnvVars(), ...opts }; const logger = getLogger(); + // Create a temp directory for any services that need it and cleanup later + const directoryToCleanup = path.join(tmpdir(), randomBytes(8).toString('hex')); + await fs.mkdir(directoryToCleanup, { recursive: true }); + if (!config.dataDirectory) { + config.dataDirectory = directoryToCleanup; + } + let anvil: Anvil | undefined; if (!config.l1RpcUrl) { @@ -427,11 +448,16 @@ export async function setup( logger.verbose('Creating and syncing a simulated prover node...'); const proverNodePrivateKey = getPrivateKeyFromIndex(2); const proverNodePrivateKeyHex: Hex = `0x${proverNodePrivateKey!.toString('hex')}`; - proverNode = await createAndSyncProverNode(proverNodePrivateKeyHex, config, aztecNode); + proverNode = await createAndSyncProverNode( + proverNodePrivateKeyHex, + config, + aztecNode, + path.join(directoryToCleanup, randomBytes(8).toString('hex')), + ); } logger.verbose('Creating a pxe...'); - const { pxe } = await setupPXEService(aztecNode!, pxeOpts, logger); + const { pxe, teardown: pxeTeardown } = await setupPXEService(aztecNode!, pxeOpts, logger); if (!config.skipProtocolContracts) { logger.verbose('Setting up Fee Juice...'); @@ -444,6 +470,8 @@ export async function setup( const cheatCodes = await CheatCodes.create(config.l1RpcUrl, pxe!); const teardown = async () => { + await pxeTeardown(); + if (aztecNode instanceof AztecNodeService) { await aztecNode?.stop(); } @@ -454,8 +482,19 @@ export async function setup( await acvmConfig.cleanup(); } + if (bbConfig?.cleanup) { + // remove the temp directory created for the acvm + logger.verbose(`Cleaning up BB state`); + await bbConfig.cleanup(); + } + await anvil?.stop(); await watcher.stop(); + + if (directoryToCleanup) { + logger.verbose(`Cleaning up data directory at ${directoryToCleanup}`); + await fs.rm(directoryToCleanup, { recursive: true, force: true }); + } }; return { @@ -659,6 +698,7 @@ export async function createAndSyncProverNode( proverNodePrivateKey: `0x${string}`, aztecNodeConfig: AztecNodeConfig, aztecNode: AztecNode, + dataDirectory: string, ) { // Disable stopping the aztec node as the prover coordination test will kill it otherwise // This is only required when stopping the prover node for testing @@ -669,7 +709,7 @@ export async function createAndSyncProverNode( }; // Creating temp store and archiver for simulated prover node - const archiverConfig = { ...aztecNodeConfig, dataDirectory: undefined }; + const archiverConfig = { ...aztecNodeConfig, dataDirectory }; const archiver = await createArchiver(archiverConfig, new NoopTelemetryClient(), { blockUntilSync: true }); // Prover node config is for simulated proofs diff --git a/yarn-project/kv-store/src/indexeddb/store.ts b/yarn-project/kv-store/src/indexeddb/store.ts index ad841c0a9ab..fe72cdf0662 100644 --- a/yarn-project/kv-store/src/indexeddb/store.ts +++ b/yarn-project/kv-store/src/indexeddb/store.ts @@ -1,6 +1,6 @@ import { type Logger } from '@aztec/foundation/log'; -import { type DBSchema, type IDBPDatabase, openDB } from 'idb'; +import { type DBSchema, type IDBPDatabase, deleteDB, openDB } from 'idb'; import { type AztecAsyncArray } from '../interfaces/array.js'; import { type Key } from '../interfaces/common.js'; @@ -183,7 +183,8 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore { /** Deletes this store and removes the database */ delete() { this.#containers.clear(); - return Promise.resolve(this.#rootDB.deleteObjectStore('data')); + this.#rootDB.close(); + return deleteDB(this.#name); } estimateSize(): { mappingSize: number; actualSize: number; numItems: number } { diff --git a/yarn-project/kv-store/src/interfaces/array_test_suite.ts b/yarn-project/kv-store/src/interfaces/array_test_suite.ts index 0affe23305e..552e7201cf1 100644 --- a/yarn-project/kv-store/src/interfaces/array_test_suite.ts +++ b/yarn-project/kv-store/src/interfaces/array_test_suite.ts @@ -20,6 +20,10 @@ export function describeAztecArray( arr = store.openArray('test'); }); + afterEach(async () => { + await store.delete(); + }); + async function length(sut: AztecAsyncArray | AztecArray = arr) { return isSyncStore(store) && !forceAsync ? (sut as AztecArray).length diff --git a/yarn-project/kv-store/src/interfaces/map_test_suite.ts b/yarn-project/kv-store/src/interfaces/map_test_suite.ts index 3c999f01b2c..7736315ec6f 100644 --- a/yarn-project/kv-store/src/interfaces/map_test_suite.ts +++ b/yarn-project/kv-store/src/interfaces/map_test_suite.ts @@ -21,6 +21,10 @@ export function describeAztecMap( map = store.openMultiMap('test'); }); + afterEach(async () => { + await store.delete(); + }); + async function get(key: Key, sut: AztecAsyncMap | AztecMap = map) { return isSyncStore(store) && !forceAsync ? (sut as AztecMultiMap).get(key) diff --git a/yarn-project/kv-store/src/interfaces/set_test_suite.ts b/yarn-project/kv-store/src/interfaces/set_test_suite.ts index 08f2758ebf3..ad2de503f31 100644 --- a/yarn-project/kv-store/src/interfaces/set_test_suite.ts +++ b/yarn-project/kv-store/src/interfaces/set_test_suite.ts @@ -21,6 +21,10 @@ export function describeAztecSet( set = store.openSet('test'); }); + afterEach(async () => { + await store.delete(); + }); + async function has(key: string) { return isSyncStore(store) && !forceAsync ? (set as AztecSet).has(key) diff --git a/yarn-project/kv-store/src/interfaces/singleton_test_suite.ts b/yarn-project/kv-store/src/interfaces/singleton_test_suite.ts index b3ad148f8d0..6ecaa24522d 100644 --- a/yarn-project/kv-store/src/interfaces/singleton_test_suite.ts +++ b/yarn-project/kv-store/src/interfaces/singleton_test_suite.ts @@ -18,6 +18,10 @@ export function describeAztecSingleton( singleton = store.openSingleton('test'); }); + afterEach(async () => { + await store.delete(); + }); + async function get() { return isSyncStore(store) && !forceAsync ? (singleton as AztecSingleton).get() diff --git a/yarn-project/kv-store/src/interfaces/store_test_suite.ts b/yarn-project/kv-store/src/interfaces/store_test_suite.ts index 91c2240ecc3..051504889f0 100644 --- a/yarn-project/kv-store/src/interfaces/store_test_suite.ts +++ b/yarn-project/kv-store/src/interfaces/store_test_suite.ts @@ -32,21 +32,25 @@ export function describeAztecStore( expect(await get(forkedStore, forkedSingleton)).to.equal('bar'); await forkedSingleton.delete(); expect(await get(store, singleton)).to.equal('foo'); + await forkedStore.delete(); }; it('forks a persistent store', async () => { const store = await getPersistentStore(); await itForks(store); + await store.delete(); }); it('forks a persistent store with no path', async () => { const store = await getPersistentNoPathStore(); await itForks(store); + await store.delete(); }); it('forks an ephemeral store', async () => { const store = await getEphemeralStore(); await itForks(store); + await store.delete(); }); }); } diff --git a/yarn-project/kv-store/src/lmdb/counter.test.ts b/yarn-project/kv-store/src/lmdb/counter.test.ts index 7e1c8524cf9..12b748fb358 100644 --- a/yarn-project/kv-store/src/lmdb/counter.test.ts +++ b/yarn-project/kv-store/src/lmdb/counter.test.ts @@ -3,8 +3,11 @@ import { toArray } from '@aztec/foundation/iterable'; import { expect, use } from 'chai'; import chaiAsPromised from 'chai-as-promised'; +import fs from 'fs/promises'; import { type Database, open } from 'lmdb'; import forEach from 'mocha-each'; +import { tmpdir } from 'os'; +import path from 'path'; import { LmdbAztecCounter } from './counter.js'; @@ -12,9 +15,18 @@ use(chaiAsPromised); describe('LmdbAztecCounter', () => { let db: Database; + let dir: string; - beforeEach(() => { - db = open({} as any); + beforeEach(async () => { + dir = path.join(tmpdir(), randomBytes(8).toString('hex')); + await fs.mkdir(dir, { recursive: true }); + db = open({ path: dir } as any); + }); + + afterEach(async () => { + await db.drop(); + await db.close(); + await fs.rm(dir, { recursive: true, force: true }); }); forEach([ diff --git a/yarn-project/kv-store/src/lmdb/store.ts b/yarn-project/kv-store/src/lmdb/store.ts index ad66ec76691..f0f453a98ad 100644 --- a/yarn-project/kv-store/src/lmdb/store.ts +++ b/yarn-project/kv-store/src/lmdb/store.ts @@ -1,9 +1,10 @@ +import { randomBytes } from '@aztec/foundation/crypto'; import { createLogger } from '@aztec/foundation/log'; import { promises as fs, mkdirSync } from 'fs'; import { type Database, type RootDatabase, open } from 'lmdb'; import { tmpdir } from 'os'; -import { dirname, join } from 'path'; +import { join } from 'path'; import { type AztecArray, type AztecAsyncArray } from '../interfaces/array.js'; import { type Key } from '../interfaces/common.js'; @@ -29,7 +30,7 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore { #multiMapData: Database; #log = createLogger('kv-store:lmdb'); - constructor(rootDb: RootDatabase, public readonly isEphemeral: boolean, private path?: string) { + constructor(rootDb: RootDatabase, public readonly isEphemeral: boolean, private path: string) { this.#rootDb = rootDb; // big bucket to store all the data @@ -64,13 +65,12 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore { ephemeral: boolean = false, log = createLogger('kv-store:lmdb'), ): AztecLmdbStore { - if (path) { - mkdirSync(path, { recursive: true }); - } + const dbPath = path ?? join(tmpdir(), randomBytes(8).toString('hex')); + mkdirSync(dbPath, { recursive: true }); const mapSize = 1024 * mapSizeKb; log.debug(`Opening LMDB database at ${path || 'temporary location'} with map size ${mapSize}`); - const rootDb = open({ path, noSync: ephemeral, mapSize }); - return new AztecLmdbStore(rootDb, ephemeral, path); + const rootDb = open({ path: dbPath, noSync: ephemeral, mapSize }); + return new AztecLmdbStore(rootDb, ephemeral, dbPath); } /** @@ -78,10 +78,9 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore { * @returns A new AztecLmdbStore. */ async fork() { - const baseDir = this.path ? dirname(this.path) : tmpdir(); + const baseDir = this.path; this.#log.debug(`Forking store with basedir ${baseDir}`); - const forkPath = - (await fs.mkdtemp(join(baseDir, 'aztec-store-fork-'))) + (this.isEphemeral || !this.path ? '/data.mdb' : ''); + const forkPath = await fs.mkdtemp(join(baseDir, 'aztec-store-fork-')); this.#log.verbose(`Forking store to ${forkPath}`); await this.#rootDb.backup(forkPath, false); const forkDb = open(forkPath, { noSync: this.isEphemeral }); diff --git a/yarn-project/kv-store/src/stores/l2_tips_store.test.ts b/yarn-project/kv-store/src/stores/l2_tips_store.test.ts index d9ec9845fc1..32010b33985 100644 --- a/yarn-project/kv-store/src/stores/l2_tips_store.test.ts +++ b/yarn-project/kv-store/src/stores/l2_tips_store.test.ts @@ -17,6 +17,10 @@ describe('L2TipsStore', () => { tipsStore = new L2TipsStore(kvStore, 'test'); }); + afterEach(async () => { + await kvStore.delete(); + }); + const makeBlock = (number: number): L2Block => ({ number, header: { hash: () => new Fr(number) } as BlockHeader } as L2Block);