diff --git a/api/clients/v2/codecs/blob.go b/api/clients/v2/coretypes/blob.go similarity index 88% rename from api/clients/v2/codecs/blob.go rename to api/clients/v2/coretypes/blob.go index c52148f679..cc34b2f3d7 100644 --- a/api/clients/v2/codecs/blob.go +++ b/api/clients/v2/coretypes/blob.go @@ -1,9 +1,10 @@ -package codecs +package coretypes import ( "fmt" "github.com/Layr-Labs/eigenda/api/clients/codecs" + "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/encoding/fft" "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/Layr-Labs/eigenda/encoding/utils/codec" @@ -27,6 +28,15 @@ type Blob struct { // DeserializeBlob initializes a Blob from bytes func DeserializeBlob(bytes []byte, blobLengthSymbols uint32) (*Blob, error) { + // we check that length of bytes is <= blob length, rather than checking for equality, because it's possible + // that the bytes being deserialized have had trailing 0s truncated. + if uint32(len(bytes)) > blobLengthSymbols*encoding.BYTES_PER_SYMBOL { + return nil, fmt.Errorf( + "length (%d bytes) is greater than claimed blob length (%d bytes)", + len(bytes), + blobLengthSymbols*encoding.BYTES_PER_SYMBOL) + } + coeffPolynomial, err := rs.ToFrArray(bytes) if err != nil { return nil, fmt.Errorf("bytes to field elements: %w", err) @@ -66,17 +76,18 @@ func (b *Blob) ToPayload(payloadForm codecs.PolynomialForm) (*Payload, error) { return payload, nil } +// BlobLengthSymbols returns the length of the blob, in symbols +func (b *Blob) BlobLengthSymbols() uint32 { + return b.blobLengthSymbols +} + // toEncodedPayload creates an encodedPayload from the blob // // The payloadForm indicates how payloads are interpreted. The way that payloads are interpreted dictates what // conversion, if any, must be performed when creating an encoded payload from the blob. func (b *Blob) toEncodedPayload(payloadForm codecs.PolynomialForm) (*encodedPayload, error) { - maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(b.blobLengthSymbols) - if err != nil { - return nil, fmt.Errorf("get max permissible payload length: %w", err) - } - var payloadElements []fr.Element + var err error switch payloadForm { case codecs.PolynomialFormCoeff: // the payload is interpreted as coefficients of the polynomial, so no conversion needs to be done, given that @@ -93,6 +104,11 @@ func (b *Blob) toEncodedPayload(payloadForm codecs.PolynomialForm) (*encodedPayl return nil, fmt.Errorf("invalid polynomial form") } + maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(b.blobLengthSymbols) + if err != nil { + return nil, fmt.Errorf("get max permissible payload length: %w", err) + } + encodedPayload, err := encodedPayloadFromElements(payloadElements, maxPermissiblePayloadLength) if err != nil { return nil, fmt.Errorf("encoded payload from elements %w", err) diff --git a/api/clients/v2/codecs/blob_test.go b/api/clients/v2/coretypes/blob_test.go similarity index 98% rename from api/clients/v2/codecs/blob_test.go rename to api/clients/v2/coretypes/blob_test.go index 7fae932e47..dac4623bd2 100644 --- a/api/clients/v2/codecs/blob_test.go +++ b/api/clients/v2/coretypes/blob_test.go @@ -1,4 +1,4 @@ -package codecs +package coretypes import ( "bytes" diff --git a/api/clients/v2/codecs/encoded_payload.go b/api/clients/v2/coretypes/encoded_payload.go similarity index 99% rename from api/clients/v2/codecs/encoded_payload.go rename to api/clients/v2/coretypes/encoded_payload.go index a8eb972563..f471b68c45 100644 --- a/api/clients/v2/codecs/encoded_payload.go +++ b/api/clients/v2/coretypes/encoded_payload.go @@ -1,4 +1,4 @@ -package codecs +package coretypes import ( "encoding/binary" diff --git a/api/clients/v2/codecs/encoded_payload_test.go b/api/clients/v2/coretypes/encoded_payload_test.go similarity index 99% rename from api/clients/v2/codecs/encoded_payload_test.go rename to api/clients/v2/coretypes/encoded_payload_test.go index 0053715bc9..2f50d6ca45 100644 --- a/api/clients/v2/codecs/encoded_payload_test.go +++ b/api/clients/v2/coretypes/encoded_payload_test.go @@ -1,4 +1,4 @@ -package codecs +package coretypes import ( "testing" diff --git a/api/clients/v2/codecs/payload.go b/api/clients/v2/coretypes/payload.go similarity index 99% rename from api/clients/v2/codecs/payload.go rename to api/clients/v2/coretypes/payload.go index df72ed6fed..3774a8b936 100644 --- a/api/clients/v2/codecs/payload.go +++ b/api/clients/v2/coretypes/payload.go @@ -1,4 +1,4 @@ -package codecs +package coretypes import ( "fmt" diff --git a/api/clients/v2/payload_disperser.go b/api/clients/v2/payload_disperser.go index 9d60bc7238..2fd2adc2e4 100644 --- a/api/clients/v2/payload_disperser.go +++ b/api/clients/v2/payload_disperser.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/Layr-Labs/eigenda/api/clients/codecs" + "github.com/Layr-Labs/eigenda/api/clients/v2/coretypes" "github.com/Layr-Labs/eigenda/api/clients/v2/verification" dispgrpc "github.com/Layr-Labs/eigenda/api/grpc/disperser/v2" "github.com/Layr-Labs/eigenda/common/geth" @@ -24,7 +24,6 @@ import ( type PayloadDisperser struct { logger logging.Logger config PayloadDisperserConfig - codec codecs.BlobCodec disperserClient DisperserClient certVerifier verification.ICertVerifier requiredQuorumsStore *RequiredQuorumsStore @@ -87,20 +86,13 @@ func BuildPayloadDisperser(log logging.Logger, payloadDispCfg PayloadDisperserCo return nil, fmt.Errorf("new cert verifier: %w", err) } - // 5 - create codec - codec, err := codecs.CreateCodec(payloadDispCfg.PayloadPolynomialForm, payloadDispCfg.PayloadEncodingVersion) - if err != nil { - return nil, err - } - - return NewPayloadDisperser(log, payloadDispCfg, codec, disperserClient, certVerifier) + return NewPayloadDisperser(log, payloadDispCfg, disperserClient, certVerifier) } // NewPayloadDisperser creates a PayloadDisperser from subcomponents that have already been constructed and initialized. func NewPayloadDisperser( logger logging.Logger, payloadDisperserConfig PayloadDisperserConfig, - codec codecs.BlobCodec, // IMPORTANT: it is permissible for the disperserClient to be configured without a prover, but operating with this // configuration puts a trust assumption on the disperser. With a nil prover, the disperser is responsible for computing // the commitments to a blob, and the PayloadDisperser doesn't have a mechanism to verify these commitments. @@ -125,7 +117,6 @@ func NewPayloadDisperser( return &PayloadDisperser{ logger: logger, config: payloadDisperserConfig, - codec: codec, disperserClient: disperserClient, certVerifier: certVerifier, requiredQuorumsStore: requiredQuorumsStore, @@ -144,14 +135,12 @@ func (pd *PayloadDisperser) SendPayload( ctx context.Context, certVerifierAddress string, // payload is the raw data to be stored on eigenDA - payload []byte, + payload *coretypes.Payload, ) (*verification.EigenDACert, error) { - - blobBytes, err := pd.codec.EncodeBlob(payload) + blob, err := payload.ToBlob(pd.config.PayloadPolynomialForm) if err != nil { - return nil, fmt.Errorf("encode payload to blob: %w", err) + return nil, fmt.Errorf("convert payload to blob: %w", err) } - pd.logger.Debug("Payload encoded to blob") timeoutCtx, cancel := context.WithTimeout(ctx, pd.config.ContractCallTimeout) defer cancel() @@ -162,9 +151,14 @@ func (pd *PayloadDisperser) SendPayload( timeoutCtx, cancel = context.WithTimeout(ctx, pd.config.DisperseBlobTimeout) defer cancel() + + // TODO (litt3): eventually, we should consider making DisperseBlob accept an actual blob object, instead of the + // serialized bytes. The operations taking place in DisperseBlob require the bytes to be converted into field + // elements anyway, so serializing the blob here is unnecessary work. This will be a larger change that affects + // many areas of code, though. blobStatus, blobKey, err := pd.disperserClient.DisperseBlob( timeoutCtx, - blobBytes, + blob.Serialize(), pd.config.BlobVersion, requiredQuorums, ) diff --git a/api/clients/v2/payload_retriever.go b/api/clients/v2/payload_retriever.go index fd3e242984..a4398a4a19 100644 --- a/api/clients/v2/payload_retriever.go +++ b/api/clients/v2/payload_retriever.go @@ -3,6 +3,7 @@ package clients import ( "context" + "github.com/Layr-Labs/eigenda/api/clients/v2/coretypes" "github.com/Layr-Labs/eigenda/api/clients/v2/verification" ) @@ -12,5 +13,5 @@ import ( // bucket instead of from EigenDA relays or nodes. type PayloadRetriever interface { // GetPayload retrieves a payload from some backend, using the provided certificate - GetPayload(ctx context.Context, eigenDACert *verification.EigenDACert) ([]byte, error) + GetPayload(ctx context.Context, eigenDACert *verification.EigenDACert) (*coretypes.Payload, error) } diff --git a/api/clients/v2/relay_payload_retriever.go b/api/clients/v2/relay_payload_retriever.go index 2971459bd9..ac25e9594f 100644 --- a/api/clients/v2/relay_payload_retriever.go +++ b/api/clients/v2/relay_payload_retriever.go @@ -6,10 +6,9 @@ import ( "fmt" "math/rand" - "github.com/Layr-Labs/eigenda/api/clients/codecs" + "github.com/Layr-Labs/eigenda/api/clients/v2/coretypes" "github.com/Layr-Labs/eigenda/api/clients/v2/verification" core "github.com/Layr-Labs/eigenda/core/v2" - "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigensdk-go/logging" "github.com/consensys/gnark-crypto/ecc/bn254" ) @@ -24,7 +23,6 @@ type RelayPayloadRetriever struct { // must be evaluated for thread safety. random *rand.Rand config RelayPayloadRetrieverConfig - codec codecs.BlobCodec relayClient RelayClient g1Srs []bn254.G1Affine } @@ -48,19 +46,11 @@ func BuildRelayPayloadRetriever( return nil, fmt.Errorf("new relay client: %w", err) } - codec, err := codecs.CreateCodec( - relayPayloadRetrieverConfig.PayloadPolynomialForm, - relayPayloadRetrieverConfig.PayloadEncodingVersion) - if err != nil { - return nil, err - } - return NewRelayPayloadRetriever( log, rand.New(rand.NewSource(rand.Int63())), relayPayloadRetrieverConfig, relayClient, - codec, g1Srs) } @@ -71,7 +61,6 @@ func NewRelayPayloadRetriever( random *rand.Rand, relayPayloadRetrieverConfig RelayPayloadRetrieverConfig, relayClient RelayClient, - codec codecs.BlobCodec, g1Srs []bn254.G1Affine) (*RelayPayloadRetriever, error) { err := relayPayloadRetrieverConfig.checkAndSetDefaults() @@ -83,7 +72,6 @@ func NewRelayPayloadRetriever( log: log, random: random, config: relayPayloadRetrieverConfig, - codec: codec, relayClient: relayClient, g1Srs: g1Srs, }, nil @@ -100,7 +88,7 @@ func NewRelayPayloadRetriever( // verified prior to calling this method. func (pr *RelayPayloadRetriever) GetPayload( ctx context.Context, - eigenDACert *verification.EigenDACert) ([]byte, error) { + eigenDACert *verification.EigenDACert) (*coretypes.Payload, error) { blobKey, err := eigenDACert.ComputeBlobKey() if err != nil { @@ -130,7 +118,9 @@ func (pr *RelayPayloadRetriever) GetPayload( for _, val := range indices { relayKey := relayKeys[val] - blob, err := pr.getBlobWithTimeout(ctx, relayKey, blobKey) + blobLengthSymbols := eigenDACert.BlobInclusionInfo.BlobCertificate.BlobHeader.Commitment.Length + + blob, err := pr.retrieveBlobWithTimeout(ctx, relayKey, blobKey, blobLengthSymbols) // if GetBlob returned an error, try calling a different relay if err != nil { pr.log.Warn( @@ -141,17 +131,14 @@ func (pr *RelayPayloadRetriever) GetPayload( continue } - if uint(len(blob)) > blobCommitments.Length*encoding.BYTES_PER_SYMBOL { - pr.log.Warn( - "received length is greater than claimed blob length", - "blobKey", blobKey.Hex(), - "relayKey", relayKey, - "receivedLengthBytes", len(blob), - "claimedLengthBytes", blobCommitments.Length*encoding.BYTES_PER_SYMBOL) - continue - } - - valid, err := verification.GenerateAndCompareBlobCommitment(pr.g1Srs, blob, blobCommitments.Commitment) + // TODO (litt3): eventually, we should make GenerateAndCompareBlobCommitment accept a blob, instead of the + // serialization of a blob. Commitment generation operates on field elements, which is how a blob is stored + // under the hood, so it's actually duplicating work to serialize the blob here. I'm declining to make this + // change now, to limit the size of the refactor PR. + valid, err := verification.GenerateAndCompareBlobCommitment( + pr.g1Srs, + blob.Serialize(), + blobCommitments.Commitment) if err != nil { pr.log.Warn( "generate and compare blob commitment", @@ -165,14 +152,13 @@ func (pr *RelayPayloadRetriever) GetPayload( continue } - payload, err := pr.codec.DecodeBlob(blob) + payload, err := blob.ToPayload(pr.config.PayloadPolynomialForm) if err != nil { pr.log.Error( - `Cert verification was successful, but decode blob failed! - This is likely a problem with the local blob codec configuration, - but could potentially indicate a maliciously generated blob certificate. - It should not be possible for an honestly generated certificate to verify - for an invalid blob!`, + `Commitment verification was successful, but conversion from blob to payload failed! + This is likely a problem with the local configuration, but could potentially indicate + malicious dispersed data. It should not be possible for a commitment to verify for an + invalid blob!`, "blobKey", blobKey.Hex(), "relayKey", relayKey, "eigenDACert", eigenDACert, "error", err) return nil, fmt.Errorf("decode blob: %w", err) } @@ -183,16 +169,30 @@ func (pr *RelayPayloadRetriever) GetPayload( return nil, fmt.Errorf("unable to retrieve blob %v from any relay. relay count: %d", blobKey.Hex(), relayKeyCount) } -// getBlobWithTimeout attempts to get a blob from a given relay, and times out based on config.FetchTimeout -func (pr *RelayPayloadRetriever) getBlobWithTimeout( +// retrieveBlobWithTimeout attempts to retrieve a blob from a given relay, and times out based on config.FetchTimeout +func (pr *RelayPayloadRetriever) retrieveBlobWithTimeout( ctx context.Context, relayKey core.RelayKey, - blobKey *core.BlobKey) ([]byte, error) { + blobKey *core.BlobKey, + // blobLengthSymbols should be taken from the eigenDACert for the blob being retrieved + blobLengthSymbols uint32, +) (*coretypes.Blob, error) { timeoutCtx, cancel := context.WithTimeout(ctx, pr.config.RelayTimeout) defer cancel() - return pr.relayClient.GetBlob(timeoutCtx, relayKey, *blobKey) + // TODO (litt3): eventually, we should make GetBlob return an actual blob object, instead of the serialized bytes. + blobBytes, err := pr.relayClient.GetBlob(timeoutCtx, relayKey, *blobKey) + if err != nil { + return nil, fmt.Errorf("get blob from relay: %w", err) + } + + blob, err := coretypes.DeserializeBlob(blobBytes, blobLengthSymbols) + if err != nil { + return nil, fmt.Errorf("deserialize blob: %w", err) + } + + return blob, nil } // Close is responsible for calling close on all internal clients. This method will do its best to close all internal diff --git a/api/clients/v2/test/relay_payload_retriever_test.go b/api/clients/v2/test/relay_payload_retriever_test.go index c11cf6f61d..0fa69b90fd 100644 --- a/api/clients/v2/test/relay_payload_retriever_test.go +++ b/api/clients/v2/test/relay_payload_retriever_test.go @@ -10,8 +10,8 @@ import ( "testing" "time" - "github.com/Layr-Labs/eigenda/api/clients/codecs" "github.com/Layr-Labs/eigenda/api/clients/v2" + "github.com/Layr-Labs/eigenda/api/clients/v2/coretypes" clientsmock "github.com/Layr-Labs/eigenda/api/clients/v2/mock" "github.com/Layr-Labs/eigenda/api/clients/v2/verification" commonv2 "github.com/Layr-Labs/eigenda/api/grpc/common/v2" @@ -20,22 +20,24 @@ import ( testrandom "github.com/Layr-Labs/eigenda/common/testutils/random" contractEigenDACertVerifier "github.com/Layr-Labs/eigenda/contracts/bindings/EigenDACertVerifier" core "github.com/Layr-Labs/eigenda/core/v2" + "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/encoding/kzg" prover2 "github.com/Layr-Labs/eigenda/encoding/kzg/prover" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/consensys/gnark-crypto/ecc/bn254" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) const g1Path = "../../../../inabox/resources/kzg/g1.point" -const payloadLength = 100 +const maxPayloadBytes = 1025 // arbitrary value type RelayPayloadRetrieverTester struct { Random *testrandom.TestRandom RelayPayloadRetriever *clients.RelayPayloadRetriever MockRelayClient *clientsmock.MockRelayClient - Codec *codecs.DefaultBlobCodec G1Srs []bn254.G1Affine + Config clients.RelayPayloadRetrieverConfig } // buildRelayPayloadRetrieverTester sets up a client with mocks necessary for testing @@ -49,11 +51,11 @@ func buildRelayPayloadRetrieverTester(t *testing.T) RelayPayloadRetrieverTester } mockRelayClient := clientsmock.MockRelayClient{} - codec := codecs.NewDefaultBlobCodec() - random := testrandom.NewTestRandom() - g1Srs, err := kzg.ReadG1Points(g1Path, 5, uint64(runtime.GOMAXPROCS(0))) + srsPointsToLoad := encoding.NextPowerOf2(codec.GetPaddedDataLength(maxPayloadBytes)) / encoding.BYTES_PER_SYMBOL + + g1Srs, err := kzg.ReadG1Points(g1Path, uint64(srsPointsToLoad), uint64(runtime.GOMAXPROCS(0))) require.NotNil(t, g1Srs) require.NoError(t, err) @@ -62,7 +64,6 @@ func buildRelayPayloadRetrieverTester(t *testing.T) RelayPayloadRetrieverTester random.Rand, clientConfig, &mockRelayClient, - &codec, g1Srs) require.NotNil(t, client) @@ -72,8 +73,8 @@ func buildRelayPayloadRetrieverTester(t *testing.T) RelayPayloadRetrieverTester Random: random, RelayPayloadRetriever: client, MockRelayClient: &mockRelayClient, - Codec: &codec, G1Srs: g1Srs, + Config: clientConfig, } } @@ -84,9 +85,11 @@ func buildBlobAndCert( relayKeys []core.RelayKey, ) (core.BlobKey, []byte, *verification.EigenDACert) { - payloadBytes := tester.Random.Bytes(payloadLength) - blobBytes, err := tester.Codec.EncodeBlob(payloadBytes) + payloadBytes := tester.Random.Bytes(tester.Random.Intn(maxPayloadBytes)) + blob, err := coretypes.NewPayload(payloadBytes).ToBlob(tester.Config.PayloadPolynomialForm) require.NoError(t, err) + + blobBytes := blob.Serialize() require.NotNil(t, blobBytes) kzgConfig := &kzg.KzgConfig{ diff --git a/api/clients/v2/validator_payload_retriever.go b/api/clients/v2/validator_payload_retriever.go index bc65c50466..d28b3d5961 100644 --- a/api/clients/v2/validator_payload_retriever.go +++ b/api/clients/v2/validator_payload_retriever.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/Layr-Labs/eigenda/api/clients/codecs" + "github.com/Layr-Labs/eigenda/api/clients/v2/coretypes" "github.com/Layr-Labs/eigenda/api/clients/v2/verification" "github.com/Layr-Labs/eigenda/common/geth" "github.com/Layr-Labs/eigenda/core" @@ -25,7 +25,6 @@ import ( type ValidatorPayloadRetriever struct { logger logging.Logger config ValidatorPayloadRetrieverConfig - codec codecs.BlobCodec retrievalClient RetrievalClient g1Srs []bn254.G1Affine } @@ -74,17 +73,9 @@ func BuildValidatorPayloadRetriever( kzgVerifier, int(validatorPayloadRetrieverConfig.MaxConnectionCount)) - codec, err := codecs.CreateCodec( - validatorPayloadRetrieverConfig.PayloadPolynomialForm, - validatorPayloadRetrieverConfig.PayloadEncodingVersion) - if err != nil { - return nil, fmt.Errorf("create codec: %w", err) - } - return &ValidatorPayloadRetriever{ logger: logger, config: validatorPayloadRetrieverConfig, - codec: codec, retrievalClient: retrievalClient, g1Srs: kzgVerifier.Srs.G1, }, nil @@ -94,7 +85,6 @@ func BuildValidatorPayloadRetriever( func NewValidatorPayloadRetriever( logger logging.Logger, config ValidatorPayloadRetrieverConfig, - codec codecs.BlobCodec, retrievalClient RetrievalClient, g1Srs []bn254.G1Affine, ) (*ValidatorPayloadRetriever, error) { @@ -106,7 +96,6 @@ func NewValidatorPayloadRetriever( return &ValidatorPayloadRetriever{ logger: logger, config: config, - codec: codec, retrievalClient: retrievalClient, g1Srs: g1Srs, }, nil @@ -120,7 +109,7 @@ func NewValidatorPayloadRetriever( func (pr *ValidatorPayloadRetriever) GetPayload( ctx context.Context, eigenDACert *verification.EigenDACert, -) ([]byte, error) { +) (*coretypes.Payload, error) { blobKey, err := eigenDACert.ComputeBlobKey() if err != nil { @@ -135,7 +124,7 @@ func (pr *ValidatorPayloadRetriever) GetPayload( // TODO (litt3): Add a feature which keeps chunks from previous quorums, and just fills in gaps for _, quorumID := range blobHeader.QuorumNumbers { - blobBytes, err := pr.getBlobWithTimeout( + blob, err := pr.retrieveBlobWithTimeout( ctx, *blobKey, blobHeader.Version, @@ -152,17 +141,11 @@ func (pr *ValidatorPayloadRetriever) GetPayload( continue } - if uint(len(blobBytes)) > commitment.Length*encoding.BYTES_PER_SYMBOL { - pr.logger.Warn( - "received length is greater than claimed blob length", - "blobKey", blobKey.Hex(), - "quorumID", quorumID, - "receivedLengthBytes", len(blobBytes), - "claimedLengthBytes", commitment.Length*encoding.BYTES_PER_SYMBOL) - continue - } - - valid, err := verification.GenerateAndCompareBlobCommitment(pr.g1Srs, blobBytes, commitment.Commitment) + // TODO (litt3): eventually, we should make GenerateAndCompareBlobCommitment accept a blob, instead of the + // serialization of a blob. Commitment generation operates on field elements, which is how a blob is stored + // under the hood, so it's actually duplicating work to serialize the blob here. I'm declining to make this + // change now, to limit the size of the refactor PR. + valid, err := verification.GenerateAndCompareBlobCommitment(pr.g1Srs, blob.Serialize(), commitment.Commitment) if err != nil { pr.logger.Warn( "generate and compare blob commitment", @@ -176,14 +159,13 @@ func (pr *ValidatorPayloadRetriever) GetPayload( continue } - payload, err := pr.codec.DecodeBlob(blobBytes) + payload, err := blob.ToPayload(pr.config.PayloadPolynomialForm) if err != nil { pr.logger.Error( - `Cert verification was successful, but decode blob failed! - This is likely a problem with the local blob codec configuration, - but could potentially indicate a maliciously generated blob certificate. - It should not be possible for an honestly generated certificate to verify - for an invalid blob!`, + `Commitment verification was successful, but conversion from blob to payload failed! + This is likely a problem with the local configuration, but could potentially indicate + malicious dispersed data. It should not be possible for a commitment to verify for an + invalid blob!`, "blobKey", blobKey.Hex(), "quorumID", quorumID, "eigenDACert", eigenDACert, "error", err) return nil, fmt.Errorf("decode blob: %w", err) } @@ -194,23 +176,35 @@ func (pr *ValidatorPayloadRetriever) GetPayload( return nil, fmt.Errorf("unable to retrieve payload from quorums %v", blobHeader.QuorumNumbers) } -// getBlobWithTimeout attempts to get a blob from a given quorum, and times out based on config.RetrievalTimeout -func (pr *ValidatorPayloadRetriever) getBlobWithTimeout( +// retrieveBlobWithTimeout attempts to retrieve a blob from a given quorum, and times out based on config.RetrievalTimeout +func (pr *ValidatorPayloadRetriever) retrieveBlobWithTimeout( ctx context.Context, blobKey corev2.BlobKey, blobVersion corev2.BlobVersion, blobCommitments encoding.BlobCommitments, referenceBlockNumber uint32, - quorumID core.QuorumID) ([]byte, error) { + quorumID core.QuorumID) (*coretypes.Blob, error) { timeoutCtx, cancel := context.WithTimeout(ctx, pr.config.RetrievalTimeout) defer cancel() - return pr.retrievalClient.GetBlob( + // TODO (litt3): eventually, we should make GetBlob return an actual blob object, instead of the serialized bytes. + blobBytes, err := pr.retrievalClient.GetBlob( timeoutCtx, blobKey, blobVersion, blobCommitments, uint64(referenceBlockNumber), quorumID) + + if err != nil { + return nil, fmt.Errorf("get blob: %w", err) + } + + blob, err := coretypes.DeserializeBlob(blobBytes, uint32(blobCommitments.Length)) + if err != nil { + return nil, fmt.Errorf("deserialize blob: %w", err) + } + + return blob, nil } diff --git a/test/v2/client/test_client.go b/test/v2/client/test_client.go index 02f009c198..292fde81dc 100644 --- a/test/v2/client/test_client.go +++ b/test/v2/client/test_client.go @@ -8,8 +8,8 @@ import ( "strings" "time" - "github.com/Layr-Labs/eigenda/api/clients/codecs" "github.com/Layr-Labs/eigenda/api/clients/v2" + "github.com/Layr-Labs/eigenda/api/clients/v2/coretypes" "github.com/Layr-Labs/eigenda/encoding" "github.com/Layr-Labs/eigenda/encoding/kzg/prover" "github.com/prometheus/client_golang/prometheus" @@ -39,6 +39,7 @@ const ( // TestClient encapsulates the various clients necessary for interacting with EigenDA. type TestClient struct { config *TestClientConfig + payloadClientConfig *clients.PayloadClientConfig logger logging.Logger disperserClient clients.DisperserClient payloadDisperser *clients.PayloadDisperser @@ -51,7 +52,6 @@ type TestClient struct { privateKey string metricsRegistry *prometheus.Registry metrics *testClientMetrics - blobCodec codecs.BlobCodec } // NewTestClient creates a new TestClient instance. @@ -154,11 +154,9 @@ func NewTestClient( return nil, fmt.Errorf("failed to create cert verifier: %w", err) } - blobCodec, err := codecs.CreateCodec(codecs.PolynomialFormEval, codecs.PayloadEncodingVersion0) - if err != nil { - return nil, fmt.Errorf("failed to create blob codec: %w", err) - } - + // TODO (litt3): the PayloadPolynomialForm field included inside this config should be tested with different + // values, rather than just using the default. Consider a testing strategy that would exercise both encoding + // options. payloadClientConfig := clients.GetDefaultPayloadClientConfig() payloadDisperserConfig := clients.PayloadDisperserConfig{ @@ -168,7 +166,6 @@ func NewTestClient( payloadDisperser, err := clients.NewPayloadDisperser( logger, payloadDisperserConfig, - blobCodec, disperserClient, certVerifier) if err != nil { @@ -230,7 +227,6 @@ func NewTestClient( rand.Rand, *relayPayloadRetrieverConfig, relayClient, - blobCodec, blobVerifier.Srs.G1) if err != nil { return nil, fmt.Errorf("failed to create relay payload retriever: %w", err) @@ -264,7 +260,6 @@ func NewTestClient( validatorPayloadRetriever, err := clients.NewValidatorPayloadRetriever( logger, *validatorPayloadRetrieverConfig, - blobCodec, retrievalClient, blobVerifier.Srs.G1) if err != nil { @@ -273,6 +268,7 @@ func NewTestClient( return &TestClient{ config: config, + payloadClientConfig: payloadClientConfig, logger: logger, disperserClient: disperserClient, payloadDisperser: payloadDisperser, @@ -285,7 +281,6 @@ func NewTestClient( privateKey: privateKey, metricsRegistry: metrics.registry, metrics: metrics, - blobCodec: blobCodec, }, nil } @@ -425,25 +420,34 @@ func (c *TestClient) DisperseAndVerify( } // read blob from a single relay (assuming success, otherwise will retry) - payloadBytesFromRelayRetriever, err := c.relayPayloadRetriever.GetPayload(ctx, eigenDACert) + payloadFromRelayRetriever, err := c.relayPayloadRetriever.GetPayload(ctx, eigenDACert) if err != nil { - return fmt.Errorf("failed to read blob from relay: %w", err) + return fmt.Errorf("failed to get payload from relay: %w", err) } + payloadBytesFromRelayRetriever := payloadFromRelayRetriever.Serialize() if !bytes.Equal(payload, payloadBytesFromRelayRetriever) { return fmt.Errorf("payloads do not match") } // read blob from a single quorum (assuming success, otherwise will retry) - payloadBytesFromValidatorRetriever, err := c.validatorPayloadRetriever.GetPayload(ctx, eigenDACert) + payloadFromValidatorRetriever, err := c.validatorPayloadRetriever.GetPayload(ctx, eigenDACert) if err != nil { - return fmt.Errorf("failed to read blob from validators: %w", err) + return fmt.Errorf("failed to get payload from validators: %w", err) } + payloadBytesFromValidatorRetriever := payloadFromValidatorRetriever.Serialize() if !bytes.Equal(payload, payloadBytesFromValidatorRetriever) { return fmt.Errorf("payloads do not match") } + blobLengthSymbols := eigenDACert.BlobInclusionInfo.BlobCertificate.BlobHeader.Commitment.Length + // read blob from ALL relays - err = c.ReadBlobFromRelays(ctx, *blobKey, eigenDACert.BlobInclusionInfo.BlobCertificate.RelayKeys, payload) + err = c.ReadBlobFromRelays( + ctx, + *blobKey, + eigenDACert.BlobInclusionInfo.BlobCertificate.RelayKeys, + payload, + blobLengthSymbols) if err != nil { return fmt.Errorf("failed to read blob from relays: %w", err) } @@ -473,11 +477,13 @@ func (c *TestClient) DisperseAndVerify( func (c *TestClient) DispersePayload( ctx context.Context, certVerifierAddress string, - payload []byte, + payloadBytes []byte, ) (*verification.EigenDACert, error) { - c.logger.Debugf("Dispersing payload of length %d", len(payload)) + c.logger.Debugf("Dispersing payload of length %d", len(payloadBytes)) start := time.Now() + payload := coretypes.NewPayload(payloadBytes) + cert, err := c.GetPayloadDisperser().SendPayload(ctx, certVerifierAddress, payload) if err != nil { @@ -493,24 +499,32 @@ func (c *TestClient) ReadBlobFromRelays( ctx context.Context, key corev2.BlobKey, relayKeys []corev2.RelayKey, - expectedPayload []byte) error { + expectedPayload []byte, + blobLengthSymbols uint32) error { for _, relayID := range relayKeys { start := time.Now() c.logger.Debugf("Reading blob from relay %d", relayID) - blobFromRelay, err := c.relayClient.GetBlob(ctx, relayID, key) + blobBytesFromRelay, err := c.relayClient.GetBlob(ctx, relayID, key) if err != nil { return fmt.Errorf("failed to read blob from relay: %w", err) } c.metrics.reportRelayReadTime(time.Since(start), relayID) - payloadBytesFromRelay, err := c.blobCodec.DecodeBlob(blobFromRelay) + blob, err := coretypes.DeserializeBlob(blobBytesFromRelay, blobLengthSymbols) + if err != nil { + return fmt.Errorf("failed to deserialize blob: %w", err) + } + + payload, err := blob.ToPayload(c.payloadClientConfig.PayloadPolynomialForm) if err != nil { return fmt.Errorf("failed to decode blob: %w", err) } + payloadBytesFromRelay := payload.Serialize() + if !bytes.Equal(payloadBytesFromRelay, expectedPayload) { return fmt.Errorf("payloads do not match") } @@ -526,7 +540,7 @@ func (c *TestClient) ReadBlobFromValidators( blobVersion corev2.BlobVersion, blobCommitments encoding.BlobCommitments, quorums []core.QuorumID, - expectedPayload []byte) error { + expectedPayloadBytes []byte) error { currentBlockNumber, err := c.indexedChainState.GetCurrentBlockNumber(ctx) if err != nil { @@ -538,7 +552,7 @@ func (c *TestClient) ReadBlobFromValidators( start := time.Now() - retrievedBlob, err := c.retrievalClient.GetBlob( + retrievedBlobBytes, err := c.retrievalClient.GetBlob( ctx, blobKey, blobVersion, @@ -551,11 +565,19 @@ func (c *TestClient) ReadBlobFromValidators( c.metrics.reportValidatorReadTime(time.Since(start), quorumID) - retrievedPayload, err := c.blobCodec.DecodeBlob(retrievedBlob) + blobLengthSymbols := uint32(blobCommitments.Length) + blob, err := coretypes.DeserializeBlob(retrievedBlobBytes, blobLengthSymbols) if err != nil { - return fmt.Errorf("failed to decode blob: %w", err) + return fmt.Errorf("failed to deserialize blob: %w", err) } - if !bytes.Equal(retrievedPayload, expectedPayload) { + + retrievedPayload, err := blob.ToPayload(c.payloadClientConfig.PayloadPolynomialForm) + if err != nil { + return fmt.Errorf("failed to convert blob to payload: %w", err) + } + + payloadBytes := retrievedPayload.Serialize() + if !bytes.Equal(payloadBytes, expectedPayloadBytes) { return fmt.Errorf("payloads do not match") } } diff --git a/test/v2/correctness/correctness_test.go b/test/v2/correctness/correctness_test.go index 561a314194..04a653eb45 100644 --- a/test/v2/correctness/correctness_test.go +++ b/test/v2/correctness/correctness_test.go @@ -7,16 +7,18 @@ import ( "testing" "time" + "github.com/Layr-Labs/eigenda/api/clients/codecs" "github.com/Layr-Labs/eigenda/api/clients/v2" + "github.com/Layr-Labs/eigenda/api/clients/v2/coretypes" "github.com/Layr-Labs/eigenda/core" auth "github.com/Layr-Labs/eigenda/core/auth/v2" "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/Layr-Labs/eigenda/test/v2/client" "github.com/docker/go-units" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/Layr-Labs/eigenda/common/testutils/random" - "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/stretchr/testify/require" ) @@ -246,13 +248,19 @@ func TestDispersalWithInvalidSignature(t *testing.T) { disperserClient, err := clients.NewDisperserClient(disperserConfig, signer, nil, nil) require.NoError(t, err) - payload := rand.VariableBytes(units.KiB, 2*units.KiB) - paddedPayload := codec.ConvertByPaddingEmptyByte(payload) + payloadBytes := rand.VariableBytes(units.KiB, 2*units.KiB) + + payload := coretypes.NewPayload(payloadBytes) + + // TODO (litt3): make the blob form configurable. Using PolynomialFormCoeff means that the data isn't being FFTed/IFFTed, + // and it is important for both modes of operation to be tested. + blob, err := payload.ToBlob(codecs.PolynomialFormCoeff) + require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - _, _, err = disperserClient.DisperseBlob(ctx, paddedPayload, 0, quorums) + _, _, err = disperserClient.DisperseBlob(ctx, blob.Serialize(), 0, quorums) require.Error(t, err) require.Contains(t, err.Error(), "error accounting blob") } diff --git a/test/v2/load/load_generator.go b/test/v2/load/load_generator.go index 3e91d6c52d..7ccbe9266d 100644 --- a/test/v2/load/load_generator.go +++ b/test/v2/load/load_generator.go @@ -152,13 +152,16 @@ func (l *LoadGenerator) submitBlob() { return } + blobLengthSymbols := eigenDACert.BlobInclusionInfo.BlobCertificate.BlobHeader.Commitment.Length + // Read the blob from the relays and validators for i := uint64(0); i < l.config.RelayReadAmplification; i++ { err = l.client.ReadBlobFromRelays( ctx, *blobKey, eigenDACert.BlobInclusionInfo.BlobCertificate.RelayKeys, - payload) + payload, + blobLengthSymbols) if err != nil { l.client.GetLogger().Errorf("failed to read blob from relays: %v", err) }