From dc8915977faa1b35b0691aaa71a5c47085d5b6ff Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Tue, 24 Sep 2024 00:20:54 +0100 Subject: [PATCH] indexer(visitor): avoid fully deserializing dynamic field ## Description Use `FieldVisitor` to extract necessary information from a dynamic field object, rather than fully expanding it, which can fail if the overall size is too large. Unfortunately, because `DynamicFieldInfo` includes a structured representation of the field name, this operation can still fail if the name alone is too large to deserialize using `BoundedVisitor`, but we can at least avoid deserializing the value. ## Test plan Build and run the indexer reader against a mainnet DB, and check for `0x5`: ``` sui$ cargo run --bin sui-indexer -- --database-url "$DB" --pool-size 10 \ json-rpc-service --rpc-client-url "https://fullnode.mainnet.sui.io:443" ``` Make the request: ``` $ curl -LX POST "https:/fullnode.mainnet.sui.io:443" \ --header 'Content-Type: application/json' \ --data-raw '{ "jsonrpc": "2.0", "method": "suix_getDynamicFields", "id": 1, "params": ["0x5"] }' | jq -C . | less -r ``` Expects a response like: ``` { "jsonrpc": "2.0", "result": { "data": [ { "name": { "type": "u64", "value": "2" }, "bcsName": "LQM2cdzDY3", "type": "DynamicField", "objectType": "0x3::sui_system_state_inner::SuiSystemStateInnerV2", "objectId": "0x5b890eaf2abcfa2ab90b77b8e6f3d5d8609586c3e583baf3dccd5af17edf48d1", "version": 339253281, "digest": "89nPMzp31fiSy39a7fg6TBzALdm3xA4Byd4hWr2QLahg" } ], "nextCursor": "0x5b890eaf2abcfa2ab90b77b8e6f3d5d8609586c3e583baf3dccd5af17edf48d1", "hasNextPage": false }, "id": 1 } ``` --- crates/sui-indexer/src/indexer_reader.rs | 94 ++++++++++--------- crates/sui-types/src/dynamic_field.rs | 4 +- crates/sui-types/src/dynamic_field/visitor.rs | 28 ++++++ 3 files changed, 79 insertions(+), 47 deletions(-) diff --git a/crates/sui-indexer/src/indexer_reader.rs b/crates/sui-indexer/src/indexer_reader.rs index 707293a436993..cd816ef6cdd1d 100644 --- a/crates/sui-indexer/src/indexer_reader.rs +++ b/crates/sui-indexer/src/indexer_reader.rs @@ -1,5 +1,7 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 + +use anyhow::anyhow; use anyhow::Result; use diesel::{ dsl::sql, sql_types::Bool, ExpressionMethods, JoinOnDsl, NullableExpressionMethods, @@ -7,12 +9,14 @@ use diesel::{ }; use itertools::Itertools; use std::sync::Arc; +use sui_types::dynamic_field::visitor as DFV; +use sui_types::object::bounded_visitor::BoundedVisitor; use tap::{Pipe, TapFallible}; use tracing::{debug, error, warn}; use fastcrypto::encoding::Encoding; use fastcrypto::encoding::Hex; -use move_core_types::annotated_value::{MoveStructLayout, MoveTypeLayout}; +use move_core_types::annotated_value::MoveStructLayout; use move_core_types::language_storage::{StructTag, TypeTag}; use sui_json_rpc_types::DisplayFieldsResponse; use sui_json_rpc_types::{Balance, Coin as SuiCoin, SuiCoinMetadata, SuiMoveValue}; @@ -29,7 +33,7 @@ use sui_types::{ base_types::{ObjectID, SuiAddress, VersionNumber}, committee::EpochId, digests::TransactionDigest, - dynamic_field::{DynamicFieldInfo, DynamicFieldType}, + dynamic_field::DynamicFieldInfo, object::{Object, ObjectRead}, sui_system_state::{sui_system_state_summary::SuiSystemStateSummary, SuiSystemStateTrait}, }; @@ -1166,49 +1170,57 @@ impl IndexerReader { )); } }; - let struct_tag: StructTag = move_object.type_().clone().into(); - let move_type_layout = self + let type_tag: TypeTag = move_object.type_().clone().into(); + let layout = self .package_resolver - .type_layout(TypeTag::Struct(Box::new(struct_tag.clone()))) + .type_layout(type_tag.clone()) .await .map_err(|e| { IndexerError::ResolveMoveStructError(format!( - "Failed to get type layout for type {}: {}", - struct_tag, e + "Failed to get type layout for type {}: {e}", + type_tag.to_canonical_display(/* with_prefix */ true), )) })?; - let MoveTypeLayout::Struct(move_struct_layout) = move_type_layout else { - return Err(IndexerError::ResolveMoveStructError( - "MoveTypeLayout is not Struct".to_string(), - )); - }; - let move_struct = move_object.to_move_struct(&move_struct_layout)?; - let (move_value, type_, object_id) = - DynamicFieldInfo::parse_move_object(&move_struct).tap_err(|e| warn!("{e}"))?; - let name_type = move_object.type_().try_extract_field_name(&type_)?; - let bcs_name = bcs::to_bytes(&move_value.clone().undecorate()).map_err(|e| { - IndexerError::SerdeError(format!( - "Failed to serialize dynamic field name {:?}: {e}", - move_value - )) - })?; + let field = DFV::FieldVisitor::deserialize(move_object.contents(), &layout) + .tap_err(|e| warn!("{e}"))?; + + let type_ = field.kind; + let name_type: TypeTag = field.name_layout.into(); + let bcs_name = field.name_bytes.to_owned(); + + let name_value = BoundedVisitor::deserialize_value(field.name_bytes, field.name_layout) + .tap_err(|e| warn!("{e}"))?; + let name = DynamicFieldName { type_: name_type, - value: SuiMoveValue::from(move_value).to_json_value(), + value: SuiMoveValue::from(name_value).to_json_value(), }; - Ok(Some(match type_ { - DynamicFieldType::DynamicObject => { - let object = self.get_object(&object_id, None).await?.ok_or( - IndexerError::UncategorizedError(anyhow::anyhow!( - "Failed to find object_id {:?} when trying to create dynamic field info", - object_id - )), - )?; - - let version = object.version(); - let digest = object.digest(); + let value_metadata = field.value_metadata().map_err(|e| { + warn!("{e}"); + IndexerError::UncategorizedError(anyhow!(e)) + })?; + + Ok(Some(match value_metadata { + DFV::ValueMetadata::DynamicField(object_type) => DynamicFieldInfo { + name, + bcs_name, + type_, + object_type: object_type.to_canonical_string(/* with_prefix */ true), + object_id: object.id(), + version: object.version(), + digest: object.digest(), + }, + + DFV::ValueMetadata::DynamicObjectField(object_id) => { + let object = self.get_object(&object_id, None).await?.ok_or_else(|| { + IndexerError::UncategorizedError(anyhow!( + "Failed to find object_id {} when trying to create dynamic field info", + object_id.to_canonical_display(/* with_prefix */ true), + )) + })?; + let object_type = object.data.type_().unwrap().clone(); DynamicFieldInfo { name, @@ -1216,20 +1228,10 @@ impl IndexerReader { type_, object_type: object_type.to_canonical_string(/* with_prefix */ true), object_id, - version, - digest, + version: object.version(), + digest: object.digest(), } } - DynamicFieldType::DynamicField => DynamicFieldInfo { - name, - bcs_name, - type_, - object_type: move_object.into_type().into_type_params()[1] - .to_canonical_string(/* with_prefix */ true), - object_id: object.id(), - version: object.version(), - digest: object.digest(), - }, })) } diff --git a/crates/sui-types/src/dynamic_field.rs b/crates/sui-types/src/dynamic_field.rs index fd7e7ebfd8c44..2e412b46c37c7 100644 --- a/crates/sui-types/src/dynamic_field.rs +++ b/crates/sui-types/src/dynamic_field.rs @@ -95,7 +95,9 @@ impl Display for DynamicFieldName { } } -#[derive(Clone, Serialize, Deserialize, JsonSchema, Ord, PartialOrd, Eq, PartialEq, Debug)] +#[derive( + Copy, Clone, Serialize, Deserialize, JsonSchema, Ord, PartialOrd, Eq, PartialEq, Debug, +)] pub enum DynamicFieldType { #[serde(rename_all = "camelCase")] DynamicField, diff --git a/crates/sui-types/src/dynamic_field/visitor.rs b/crates/sui-types/src/dynamic_field/visitor.rs index 21a85aca6b5e2..e8bfe6388aabf 100644 --- a/crates/sui-types/src/dynamic_field/visitor.rs +++ b/crates/sui-types/src/dynamic_field/visitor.rs @@ -5,6 +5,7 @@ use move_core_types::{ account_address::AccountAddress, annotated_value as A, annotated_visitor::{self, StructDriver, ValueDriver, VariantDriver, VecDriver, Visitor}, + language_storage::TypeTag, u256::U256, }; @@ -26,11 +27,19 @@ pub struct Field<'b, 'l> { pub value_bytes: &'b [u8], } +pub enum ValueMetadata { + DynamicField(TypeTag), + DynamicObjectField(ObjectID), +} + #[derive(thiserror::Error, Debug)] pub enum Error { #[error("Not a dynamic field")] NotADynamicField, + #[error("Not a dynamic object field")] + NotADynamicObjectField, + #[error("{0}")] Visitor(#[from] annotated_visitor::Error), } @@ -46,6 +55,25 @@ impl FieldVisitor { } } +impl<'b, 'l> Field<'b, 'l> { + /// If this field is a dynamic field, returns its value's type. If it is a dynamic object + /// field, it returns the ID of the object the value points to (which must be fetched to + /// extract its type). + pub fn value_metadata(&self) -> Result { + match self.kind { + DynamicFieldType::DynamicField => Ok(ValueMetadata::DynamicField(TypeTag::from( + self.value_layout, + ))), + + DynamicFieldType::DynamicObject => { + let id: ObjectID = + bcs::from_bytes(self.value_bytes).map_err(|_| Error::NotADynamicObjectField)?; + Ok(ValueMetadata::DynamicObjectField(id)) + } + } + } +} + impl<'b, 'l> Visitor<'b, 'l> for FieldVisitor { type Value = Field<'b, 'l>; type Error = Error;