feat: track block production selection results and payload values (#7203)

* feat: track block production selection source and reason

* Rename reason to builder_censorship

* Fix metric name

* Add metric to track execution payload value

* Update metric descriptions

* Remove ETH suffix from payload values

* Rename metric

* Update import path

* Return block selection result

* Reorder if statements

Author: nflaig

packages/beacon-node/src/api/impl/validator/index.ts

@@ -47,7 +47,15 @@ import {
   getValidatorStatus,
 } from "@lodestar/types";
 import {ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice";
-import {fromHex, toHex, resolveOrRacePromises, prettyWeiToEth, toRootHex} from "@lodestar/utils";
+import {
+  fromHex,
+  toHex,
+  resolveOrRacePromises,
+  prettyWeiToEth,
+  toRootHex,
+  TimeoutError,
+  formatWeiToEth,
+} from "@lodestar/utils";
 import {
   AttestationError,
   AttestationErrorCode,
@@ -115,6 +123,41 @@ type ProduceFullOrBlindedBlockOrContentsRes = {executionPayloadSource: ProducedB
   | (ProduceBlindedBlockRes & {executionPayloadBlinded: true})
 );
 
+/**
+ * Engine block selection reasons tracked in metrics
+ */
+export enum EngineBlockSelectionReason {
+  BuilderDisabled = "builder_disabled",
+  BuilderError = "builder_error",
+  BuilderTimeout = "builder_timeout",
+  BuilderPending = "builder_pending",
+  BuilderNoBid = "builder_no_bid",
+  BuilderCensorship = "builder_censorship",
+  BlockValue = "block_value",
+  EnginePreferred = "engine_preferred",
+}
+
+/**
+ * Builder block selection reasons tracked in metrics
+ */
+export enum BuilderBlockSelectionReason {
+  EngineDisabled = "engine_disabled",
+  EngineError = "engine_error",
+  EnginePending = "engine_pending",
+  BlockValue = "block_value",
+  BuilderPreferred = "builder_preferred",
+}
+
+export type BlockSelectionResult =
+  | {
+      source: ProducedBlockSource.engine;
+      reason: EngineBlockSelectionReason;
+    }
+  | {
+      source: ProducedBlockSource.builder;
+      reason: BuilderBlockSelectionReason;
+    };
+
 /**
  * Server implementation for handling validator duties.
  * See `@lodestar/validator/src/api` for the client implementation).
@@ -417,6 +460,7 @@ export function getValidatorApi(
 
       metrics?.blockProductionSuccess.inc({source});
       metrics?.blockProductionNumAggregated.observe({source}, block.body.attestations.length);
+      metrics?.blockProductionExecutionPayloadValue.observe({source}, Number(formatWeiToEth(executionPayloadValue)));
       logger.verbose("Produced blinded block", {
         slot,
         executionPayloadValue,
@@ -491,6 +535,7 @@ export function getValidatorApi(
 
       metrics?.blockProductionSuccess.inc({source});
       metrics?.blockProductionNumAggregated.observe({source}, block.body.attestations.length);
+      metrics?.blockProductionExecutionPayloadValue.observe({source}, Number(formatWeiToEth(executionPayloadValue)));
       logger.verbose("Produced execution block", {
         slot,
         executionPayloadValue,
@@ -694,6 +739,11 @@ export function getValidatorApi(
         ...getBlockValueLogInfo(engine.value),
       });
 
+      metrics?.blockProductionSelectionResults.inc({
+        source: ProducedBlockSource.engine,
+        reason: EngineBlockSelectionReason.BuilderCensorship,
+      });
+
       return {...engine.value, executionPayloadBlinded: false, executionPayloadSource: ProducedBlockSource.engine};
     }
 
@@ -704,6 +754,16 @@ export function getValidatorApi(
         ...getBlockValueLogInfo(builder.value),
       });
 
+      metrics?.blockProductionSelectionResults.inc({
+        source: ProducedBlockSource.builder,
+        reason:
+          isEngineEnabled === false
+            ? BuilderBlockSelectionReason.EngineDisabled
+            : engine.status === "pending"
+              ? BuilderBlockSelectionReason.EnginePending
+              : BuilderBlockSelectionReason.EngineError,
+      });
+
       return {...builder.value, executionPayloadBlinded: true, executionPayloadSource: ProducedBlockSource.builder};
     }
 
@@ -714,16 +774,33 @@ export function getValidatorApi(
         ...getBlockValueLogInfo(engine.value),
       });
 
+      metrics?.blockProductionSelectionResults.inc({
+        source: ProducedBlockSource.engine,
+        reason:
+          isBuilderEnabled === false
+            ? EngineBlockSelectionReason.BuilderDisabled
+            : builder.status === "pending"
+              ? EngineBlockSelectionReason.BuilderPending
+              : builder.reason instanceof NoBidReceived
+                ? EngineBlockSelectionReason.BuilderNoBid
+                : builder.reason instanceof TimeoutError
+                  ? EngineBlockSelectionReason.BuilderTimeout
+                  : EngineBlockSelectionReason.BuilderError,
+      });
+
       return {...engine.value, executionPayloadBlinded: false, executionPayloadSource: ProducedBlockSource.engine};
     }
 
     if (engine.status === "fulfilled" && builder.status === "fulfilled") {
-      const executionPayloadSource = selectBlockProductionSource({
+      const result = selectBlockProductionSource({
         builderBlockValue: builder.value.executionPayloadValue + builder.value.consensusBlockValue,
         engineBlockValue: engine.value.executionPayloadValue + engine.value.consensusBlockValue,
         builderBoostFactor,
         builderSelection,
       });
+      const executionPayloadSource = result.source;
+
+      metrics?.blockProductionSelectionResults.inc(result);
 
       logger.info(`Selected ${executionPayloadSource} block`, {
         ...loggerContext,

packages/beacon-node/src/api/impl/validator/utils.ts

@@ -3,6 +3,7 @@ import {ATTESTATION_SUBNET_COUNT} from "@lodestar/params";
 import {routes} from "@lodestar/api";
 import {BLSPubkey, CommitteeIndex, ProducedBlockSource, Slot, ValidatorIndex} from "@lodestar/types";
 import {MAX_BUILDER_BOOST_FACTOR} from "@lodestar/validator";
+import {BlockSelectionResult, BuilderBlockSelectionReason, EngineBlockSelectionReason} from "./index.js";
 
 export function computeSubnetForCommitteesAtSlot(
   slot: Slot,
@@ -54,21 +55,31 @@ export function selectBlockProductionSource({
   engineBlockValue: bigint;
   builderBlockValue: bigint;
   builderBoostFactor: bigint;
-}): ProducedBlockSource {
+}): BlockSelectionResult {
   switch (builderSelection) {
     case routes.validator.BuilderSelection.ExecutionAlways:
     case routes.validator.BuilderSelection.ExecutionOnly:
-      return ProducedBlockSource.engine;
+      return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.EnginePreferred};
 
     case routes.validator.BuilderSelection.Default:
-    case routes.validator.BuilderSelection.MaxProfit:
-      return builderBoostFactor !== MAX_BUILDER_BOOST_FACTOR &&
-        (builderBoostFactor === BigInt(0) || engineBlockValue >= (builderBlockValue * builderBoostFactor) / BigInt(100))
-        ? ProducedBlockSource.engine
-        : ProducedBlockSource.builder;
+    case routes.validator.BuilderSelection.MaxProfit: {
+      if (builderBoostFactor === BigInt(0)) {
+        return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.EnginePreferred};
+      }
+
+      if (builderBoostFactor === MAX_BUILDER_BOOST_FACTOR) {
+        return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BuilderPreferred};
+      }
+
+      if (engineBlockValue >= (builderBlockValue * builderBoostFactor) / BigInt(100)) {
+        return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.BlockValue};
+      }
+
+      return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BlockValue};
+    }
 
     case routes.validator.BuilderSelection.BuilderAlways:
     case routes.validator.BuilderSelection.BuilderOnly:
-      return ProducedBlockSource.builder;
+      return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BuilderPreferred};
   }
 }

packages/beacon-node/src/metrics/metrics/beacon.ts

@@ -3,6 +3,11 @@ import {NotReorgedReason} from "@lodestar/fork-choice/lib/forkChoice/interface.j
 import {UpdateHeadOpt} from "@lodestar/fork-choice";
 import {RegistryMetricCreator} from "../utils/registryMetricCreator.js";
 import {BlockProductionStep, PayloadPreparationType} from "../../chain/produceBlock/index.js";
+import {
+  BlockSelectionResult,
+  BuilderBlockSelectionReason,
+  EngineBlockSelectionReason,
+} from "../../api/impl/validator/index.js";
 
 export type BeaconMetrics = ReturnType<typeof createBeaconMetrics>;
 
@@ -160,12 +165,23 @@ export function createBeaconMetrics(register: RegistryMetricCreator) {
       help: "Count of blocks successfully produced",
       labelNames: ["source"],
     }),
+    blockProductionSelectionResults: register.gauge<BlockSelectionResult>({
+      name: "beacon_block_production_selection_results_total",
+      help: "Count of all block production selection results",
+      labelNames: ["source", "reason"],
+    }),
     blockProductionNumAggregated: register.histogram<{source: ProducedBlockSource}>({
       name: "beacon_block_production_num_aggregated_total",
       help: "Count of all aggregated attestations in our produced block",
       buckets: [32, 64, 96, 128],
       labelNames: ["source"],
     }),
+    blockProductionExecutionPayloadValue: register.histogram<{source: ProducedBlockSource}>({
+      name: "beacon_block_production_execution_payload_value",
+      help: "Execution payload value denominated in ETH of produced blocks",
+      buckets: [0.001, 0.005, 0.01, 0.03, 0.05, 0.07, 0.1, 0.3, 0.5, 1],
+      labelNames: ["source"],
+    }),
 
     blockProductionCaches: {
       producedBlockRoot: register.gauge({

packages/utils/src/format.ts

@@ -44,11 +44,18 @@ export function formatBigDecimal(numerator: bigint, denominator: bigint, maxDeci
 // display upto 5 decimal places
 const MAX_DECIMAL_FACTOR = BigInt("100000");
 
+/**
+ * Format wei as ETH, with up to 5 decimals
+ */
+export function formatWeiToEth(wei: bigint): string {
+  return formatBigDecimal(wei, ETH_TO_WEI, MAX_DECIMAL_FACTOR);
+}
+
 /**
  * Format wei as ETH, with up to 5 decimals and append ' ETH'
  */
 export function prettyWeiToEth(wei: bigint): string {
-  return `${formatBigDecimal(wei, ETH_TO_WEI, MAX_DECIMAL_FACTOR)} ETH`;
+  return `${formatWeiToEth(wei)} ETH`;
 }
 
 /**