chore: Clean up data configuration

cspell.json

@@ -120,6 +120,7 @@
     "ierc",
     "indexeddb",
     "initialise",
+    "initialising",
     "interruptible",
     "isequal",
     "ivpk",
@@ -154,6 +155,7 @@
     "monomorphize",
     "mplex",
     "msgpack",
+    "msgpackr",
     "muldiv",
     "multiaddr",
     "multiaddrs",
@@ -163,6 +165,7 @@
     "muxers",
     "nada",
     "namespacing",
+    "napi",
     "Nargo",
     "nixpkgs",
     "nodebuffer",

yarn-project/archiver/src/archiver/config.ts

@@ -38,11 +38,6 @@ export type ArchiverConfig = {
    */
   l1Contracts: L1ContractAddresses;
 
-  /**
-   * Optional dir to store data. If omitted will store in memory.
-   */
-  dataDirectory: string | undefined;
-
   /** The max number of logs that can be obtained in 1 "getUnencryptedLogs" call. */
   maxLogs?: number;
 } & L1ReaderConfig &
@@ -57,11 +52,7 @@ export const archiverConfigMappings: ConfigMappingsType<ArchiverConfig> = {
   archiverPollingIntervalMS: {
     env: 'ARCHIVER_POLLING_INTERVAL_MS',
     description: 'The polling interval in ms for retrieving new L2 blocks and encrypted logs.',
-    ...numberConfigHelper(1000),
-  },
-  dataDirectory: {
-    env: 'DATA_DIRECTORY',
-    description: 'Optional dir to store data. If omitted will store in memory.',
+    ...numberConfigHelper(1_000),
   },
   maxLogs: {
     env: 'ARCHIVER_MAX_LOGS',

yarn-project/archiver/src/factory.ts

@@ -2,6 +2,7 @@ import { type ArchiverApi, type Service } from '@aztec/circuit-types';
 import { type ContractClassPublic } from '@aztec/circuits.js';
 import { createDebugLogger } from '@aztec/foundation/log';
 import { type Maybe } from '@aztec/foundation/types';
+import { type DataStoreConfig } from '@aztec/kv-store/config';
 import { createStore } from '@aztec/kv-store/utils';
 import { getCanonicalProtocolContract, protocolContractNames } from '@aztec/protocol-contracts';
 import { type TelemetryClient } from '@aztec/telemetry-client';
@@ -13,7 +14,7 @@ import { KVArchiverDataStore } from './archiver/index.js';
 import { createArchiverClient } from './rpc/index.js';
 
 export async function createArchiver(
-  config: ArchiverConfig,
+  config: ArchiverConfig & DataStoreConfig,
   telemetry: TelemetryClient = new NoopTelemetryClient(),
   opts: { blockUntilSync: boolean } = { blockUntilSync: true },
 ): Promise<ArchiverApi & Maybe<Service>> {

yarn-project/aztec-node/src/aztec-node/config.ts

@@ -1,5 +1,6 @@
 import { type ArchiverConfig, archiverConfigMappings } from '@aztec/archiver';
 import { type ConfigMappingsType, booleanConfigHelper, getConfigFromMappings } from '@aztec/foundation/config';
+import { type DataStoreConfig, dataConfigMappings } from '@aztec/kv-store/config';
 import { type P2PConfig, p2pConfigMappings } from '@aztec/p2p';
 import { type ProverClientConfig, proverClientConfigMappings } from '@aztec/prover-client';
 import { type SequencerClientConfig, sequencerClientConfigMappings } from '@aztec/sequencer-client';
@@ -24,7 +25,7 @@ export type AztecNodeConfig = ArchiverConfig &
   P2PConfig & {
     /** Whether the validator is disabled for this node */
     disableValidator: boolean;
-  };
+  } & DataStoreConfig;
 
 export const aztecNodeConfigMappings: ConfigMappingsType<AztecNodeConfig> = {
   ...archiverConfigMappings,
@@ -33,6 +34,7 @@ export const aztecNodeConfigMappings: ConfigMappingsType<AztecNodeConfig> = {
   ...proverClientConfigMappings,
   ...worldStateConfigMappings,
   ...p2pConfigMappings,
+  ...dataConfigMappings,
   disableValidator: {
     env: 'VALIDATOR_DISABLED',
     description: 'Whether the validator is disabled for this node.',

yarn-project/aztec/src/cli/aztec_start_options.ts

@@ -218,12 +218,6 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = {
       defaultValue: undefined,
       envVar: undefined,
     },
-    {
-      flag: '--pxe.dataDirectory <value>',
-      description: 'Where to store PXE data. If not set, will store in memory',
-      defaultValue: undefined,
-      envVar: 'PXE_DATA_DIRECTORY',
-    },
     ...getOptions('pxe', allPxeConfigMappings),
   ],
   ARCHIVER: [

yarn-project/aztec/src/cli/cmds/start_archiver.ts

@@ -2,6 +2,7 @@ import { Archiver, type ArchiverConfig, KVArchiverDataStore, archiverConfigMappi
 import { createDebugLogger } from '@aztec/aztec.js';
 import { ArchiverApiSchema } from '@aztec/circuit-types';
 import { type NamespacedApiHandlers } from '@aztec/foundation/json-rpc/server';
+import { type DataStoreConfig, dataConfigMappings } from '@aztec/kv-store/config';
 import { createStore } from '@aztec/kv-store/utils';
 import {
   createAndStartTelemetryClient,
@@ -16,7 +17,14 @@ export async function startArchiver(
   signalHandlers: (() => Promise<void>)[],
   services: NamespacedApiHandlers,
 ) {
-  const archiverConfig = extractRelevantOptions<ArchiverConfig>(options, archiverConfigMappings, 'archiver');
+  const archiverConfig = extractRelevantOptions<ArchiverConfig & DataStoreConfig>(
+    options,
+    {
+      ...archiverConfigMappings,
+      ...dataConfigMappings,
+    },
+    'archiver',
+  );
 
   const storeLog = createDebugLogger('aztec:archiver:lmdb');
   const store = await createStore('archiver', archiverConfig, storeLog);

yarn-project/end-to-end/src/benchmarks/bench_prover.test.ts

@@ -5,7 +5,7 @@ import { BBCircuitVerifier } from '@aztec/bb-prover';
 import { CompleteAddress, Fq, Fr, GasSettings } from '@aztec/circuits.js';
 import { FPCContract, FeeJuiceContract, TestContract, TokenContract } from '@aztec/noir-contracts.js';
 import { ProtocolContractAddress } from '@aztec/protocol-contracts';
-import { type PXEService, createPXEService } from '@aztec/pxe';
+import { type PXEService, type PXEServiceConfig, createPXEService } from '@aztec/pxe';
 
 import { jest } from '@jest/globals';
 
@@ -142,17 +142,18 @@ describe('benchmarks/proving', () => {
     ctx.logger.info('Starting PXEs configured with real proofs');
     provingPxes = [];
     for (let i = 0; i < 4; i++) {
-      const pxe = await createPXEService(
-        ctx.aztecNode,
-        {
-          proverEnabled: true,
-          bbBinaryPath: bbConfig.bbBinaryPath,
-          bbWorkingDirectory: bbConfig.bbWorkingDirectory,
-          l2BlockPollingIntervalMS: 1000,
-          l2StartingBlock: 1,
-        },
-        `proving-pxe-${i}`,
-      );
+      const l1Contracts = await ctx.aztecNode.getL1ContractAddresses();
+      const pxeConfig = {
+        proverEnabled: true,
+        bbBinaryPath: bbConfig.bbBinaryPath,
+        bbWorkingDirectory: bbConfig.bbWorkingDirectory,
+        l2BlockPollingIntervalMS: 1000,
+        l2StartingBlock: 1,
+        dataDirectory: undefined,
+        dataStoreMapSizeKB: 1024 * 1024,
+        l1Contracts,
+      } as PXEServiceConfig;
+      const pxe = await createPXEService(ctx.aztecNode, pxeConfig, `proving-pxe-${i}`);
 
       await getSchnorrAccount(pxe, schnorrWalletEncKey, schnorrWalletSigningKey, schnorrWalletSalt).register();
       await pxe.registerContract(initialTokenContract);

yarn-project/end-to-end/src/benchmarks/utils.ts

@@ -3,7 +3,7 @@ import { type AztecNode, BatchCall, INITIAL_L2_BLOCK_NUM, type SentTx, retryUnti
 import { times } from '@aztec/foundation/collection';
 import { randomInt } from '@aztec/foundation/crypto';
 import { BenchmarkingContract } from '@aztec/noir-contracts.js/Benchmarking';
-import { type PXEService, createPXEService } from '@aztec/pxe';
+import { type PXEService, type PXEServiceConfig, createPXEService } from '@aztec/pxe';
 
 import { mkdirpSync } from 'fs-extra';
 import { globSync } from 'glob';
@@ -102,10 +102,15 @@ export async function waitNewPXESynced(
   contract: BenchmarkingContract,
   startingBlock: number = INITIAL_L2_BLOCK_NUM,
 ): Promise<PXEService> {
-  const pxe = await createPXEService(node, {
-    l2BlockPollingIntervalMS: 100,
+  const l1Contracts = await node.getL1ContractAddresses();
+  const pxeConfig = {
     l2StartingBlock: startingBlock,
-  });
+    l2BlockPollingIntervalMS: 100,
+    dataDirectory: undefined,
+    dataStoreMapSizeKB: 1024 * 1024,
+    l1Contracts,
+  } as PXEServiceConfig;
+  const pxe = await createPXEService(node, pxeConfig);
   await pxe.registerContract(contract);
   await retryUntil(() => pxe.isGlobalStateSynchronized(), 'pxe-global-sync');
   return pxe;

yarn-project/foundation/src/config/env_var.ts

@@ -37,6 +37,7 @@ export type EnvVar =
   | 'BOT_STOP_WHEN_UNHEALTHY'
   | 'COINBASE'
   | 'DATA_DIRECTORY'
+  | 'DATA_STORE_MAP_SIZE_KB'
   | 'DEBUG'
   | 'DEPLOY_AZTEC_CONTRACTS_SALT'
   | 'DEPLOY_AZTEC_CONTRACTS'
@@ -112,7 +113,6 @@ export type EnvVar =
   | 'PROVER_REQUIRED_CONFIRMATIONS'
   | 'PROVER_TEST_DELAY_MS'
   | 'PXE_BLOCK_POLLING_INTERVAL_MS'
-  | 'PXE_DATA_DIRECTORY'
   | 'PXE_L2_STARTING_BLOCK'
   | 'PXE_PROVER_ENABLED'
   | 'QUOTE_PROVIDER_BASIS_POINT_FEE'
@@ -152,6 +152,7 @@ export type EnvVar =
   | 'PROVER_VIEM_POLLING_INTERVAL_MS'
   | 'SEQ_VIEM_POLLING_INTERVAL_MS'
   | 'WS_DB_MAP_SIZE_KB'
+  | 'WS_DATA_DIRECTORY'
   | 'ETHEREUM_SLOT_DURATION'
   | 'AZTEC_SLOT_DURATION'
   | 'AZTEC_EPOCH_DURATION'

yarn-project/kv-store/package.json

@@ -5,7 +5,8 @@
   "exports": {
     ".": "./dest/interfaces/index.js",
     "./lmdb": "./dest/lmdb/index.js",
-    "./utils": "./dest/utils.js"
+    "./utils": "./dest/utils.js",
+    "./config": "./dest/config.js"
   },
   "scripts": {
     "build": "yarn clean && tsc -b",
@@ -55,6 +56,7 @@
     ]
   },
   "dependencies": {
+    "@aztec/ethereum": "workspace:^",
     "@aztec/foundation": "workspace:^",
     "lmdb": "^3.0.6"
   },

yarn-project/kv-store/src/config.ts

@@ -0,0 +1,34 @@
+import { l1ContractAddressesMapping } from '@aztec/ethereum';
+import { type ConfigMappingsType, getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config';
+import { type EthAddress } from '@aztec/foundation/eth-address';
+
+export type DataStoreConfig = {
+  dataDirectory: string | undefined;
+  dataStoreMapSizeKB: number;
+  l1Contracts: { rollupAddress: EthAddress };
+};
+
+export const dataConfigMappings: ConfigMappingsType<DataStoreConfig> = {
+  dataDirectory: {
+    env: 'DATA_DIRECTORY',
+    description: 'Optional dir to store data. If omitted will store in memory.',
+  },
+  dataStoreMapSizeKB: {
+    env: 'DATA_STORE_MAP_SIZE_KB',
+    description: 'DB mapping size to be applied to all key/value stores',
+    ...numberConfigHelper(128 * 1_024 * 1_024), // Defaulted to 128 GB
+  },
+  l1Contracts: {
+    description: 'The deployed L1 contract addresses',
+    defaultValue: l1ContractAddressesMapping,
+  },
+};
+
+/**
+ * Returns the archiver configuration from the environment variables.
+ * Note: If an environment variable is not set, the default value is used.
+ * @returns The archiver configuration.
+ */
+export function getDataConfigFromEnv(): DataStoreConfig {
+  return getConfigFromMappings<DataStoreConfig>(dataConfigMappings);
+}

yarn-project/kv-store/src/lmdb/store.test.ts

@@ -4,6 +4,8 @@ import { join } from 'path';
 
 import { AztecLmdbStore } from './store.js';
 
+const defaultMapSize = 1024 * 1024 * 1024 * 10;
+
 describe('AztecLmdbStore', () => {
   const itForks = async (store: AztecLmdbStore) => {
     const singleton = store.openSingleton('singleton');
@@ -21,17 +23,17 @@ describe('AztecLmdbStore', () => {
 
   it('forks a persistent store', async () => {
     const path = await mkdtemp(join(tmpdir(), 'aztec-store-test-'));
-    const store = AztecLmdbStore.open(path, false);
+    const store = AztecLmdbStore.open(path, defaultMapSize, false);
     await itForks(store);
   });
 
   it('forks a persistent store with no path', async () => {
-    const store = AztecLmdbStore.open(undefined, false);
+    const store = AztecLmdbStore.open(undefined, defaultMapSize, false);
     await itForks(store);
   });
 
   it('forks an ephemeral store', async () => {
-    const store = AztecLmdbStore.open(undefined, true);
+    const store = AztecLmdbStore.open(undefined, defaultMapSize, true);
     await itForks(store);
   });
 });

yarn-project/kv-store/src/lmdb/store.ts

@@ -58,14 +58,16 @@ export class AztecLmdbStore implements AztecKVStore {
    */
   static open(
     path?: string,
+    mapSizeKb = 1 * 1024 * 1024, // defaults to 1 GB map size
     ephemeral: boolean = false,
     log = createDebugLogger('aztec:kv-store:lmdb'),
   ): AztecLmdbStore {
-    log.debug(`Opening LMDB database at ${path || 'temporary location'}`);
     if (path) {
       mkdirSync(path, { recursive: true });
     }
-    const rootDb = open({ path, noSync: ephemeral });
+    const mapSize = 1024 * mapSizeKb;
+    log.debug(`Opening LMDB database at ${path || 'temporary location'} with map size ${mapSize}`);
+    const rootDb = open({ path, noSync: ephemeral, mapSize });
     return new AztecLmdbStore(rootDb, ephemeral, path);
   }
 

yarn-project/kv-store/src/utils.ts

@@ -3,11 +3,10 @@ import { type Logger, createDebugLogger } from '@aztec/foundation/log';
 
 import { join } from 'path';
 
+import { type DataStoreConfig } from './config.js';
 import { type AztecKVStore } from './interfaces/store.js';
 import { AztecLmdbStore } from './lmdb/store.js';
 
-export type DataStoreConfig = { dataDirectory: string | undefined; l1Contracts: { rollupAddress: EthAddress } };
-
 export function createStore(name: string, config: DataStoreConfig, log: Logger = createDebugLogger('aztec:kv-store')) {
   let { dataDirectory } = config;
   if (typeof dataDirectory !== 'undefined') {
@@ -16,10 +15,14 @@ export function createStore(name: string, config: DataStoreConfig, log: Logger =
 
   log.info(
     dataDirectory
-      ? `Creating ${name} data store at directory ${dataDirectory}`
-      : `Creating ${name} ephemeral data store`,
+      ? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKB} KB`
+      : `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKB} KB`,
+  );
+  return initStoreForRollup(
+    AztecLmdbStore.open(dataDirectory, config.dataStoreMapSizeKB, false),
+    config.l1Contracts.rollupAddress,
+    log,
   );
-  return initStoreForRollup(AztecLmdbStore.open(dataDirectory, false), config.l1Contracts.rollupAddress, log);
 }
 
 /**
@@ -60,5 +63,6 @@ async function initStoreForRollup<T extends AztecKVStore>(
  * @returns A new store
  */
 export function openTmpStore(ephemeral: boolean = false): AztecKVStore {
-  return AztecLmdbStore.open(undefined, ephemeral);
+  const mapSize = 1024 * 1024 * 10; // 10 GB map size
+  return AztecLmdbStore.open(undefined, mapSize, ephemeral);
 }

yarn-project/kv-store/tsconfig.json

@@ -8,6 +8,9 @@
   "references": [
     {
       "path": "../foundation"
+    },
+    {
+      "path": "../ethereum"
     }
   ],
   "include": ["src"]

yarn-project/p2p/src/client/index.ts

@@ -1,7 +1,8 @@
 import type { ClientProtocolCircuitVerifier, L2BlockSource, WorldStateSynchronizer } from '@aztec/circuit-types';
 import { createDebugLogger } from '@aztec/foundation/log';
 import { type AztecKVStore } from '@aztec/kv-store';
-import { type DataStoreConfig, createStore } from '@aztec/kv-store/utils';
+import { type DataStoreConfig } from '@aztec/kv-store/config';
+import { createStore } from '@aztec/kv-store/utils';
 import { type TelemetryClient } from '@aztec/telemetry-client';
 import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';