Use payload and blob structs

api/clients/v2/codecs/blob.go → api/clients/v2/coretypes/blob.go

@@ -1,9 +1,10 @@
-package codecs
+package coretypes
 
 import (
 	"fmt"
 
 	"github.com/Layr-Labs/eigenda/api/clients/codecs"
+	"github.com/Layr-Labs/eigenda/encoding"
 	"github.com/Layr-Labs/eigenda/encoding/fft"
 	"github.com/Layr-Labs/eigenda/encoding/rs"
 	"github.com/Layr-Labs/eigenda/encoding/utils/codec"
@@ -27,6 +28,15 @@ type Blob struct {
 
 // DeserializeBlob initializes a Blob from bytes
 func DeserializeBlob(bytes []byte, blobLengthSymbols uint32) (*Blob, error) {
+	// we check that length of bytes is <= blob length, rather than checking for equality, because it's possible
+	// that the bytes being deserialized have had trailing 0s truncated.
+	if uint32(len(bytes)) > blobLengthSymbols*encoding.BYTES_PER_SYMBOL {
+		return nil, fmt.Errorf(
+			"length (%d bytes) is greater than claimed blob length (%d bytes)",
+			len(bytes),
+			blobLengthSymbols*encoding.BYTES_PER_SYMBOL)
+	}
+
 	coeffPolynomial, err := rs.ToFrArray(bytes)
 	if err != nil {
 		return nil, fmt.Errorf("bytes to field elements: %w", err)
@@ -66,17 +76,18 @@ func (b *Blob) ToPayload(payloadForm codecs.PolynomialForm) (*Payload, error) {
 	return payload, nil
 }
 
+// BlobLengthSymbols returns the length of the blob, in symbols
+func (b *Blob) BlobLengthSymbols() uint32 {
+	return b.blobLengthSymbols
+}
+
 // toEncodedPayload creates an encodedPayload from the blob
 //
 // The payloadForm indicates how payloads are interpreted. The way that payloads are interpreted dictates what
 // conversion, if any, must be performed when creating an encoded payload from the blob.
 func (b *Blob) toEncodedPayload(payloadForm codecs.PolynomialForm) (*encodedPayload, error) {
-	maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(b.blobLengthSymbols)
-	if err != nil {
-		return nil, fmt.Errorf("get max permissible payload length: %w", err)
-	}
-
 	var payloadElements []fr.Element
+	var err error
 	switch payloadForm {
 	case codecs.PolynomialFormCoeff:
 		// the payload is interpreted as coefficients of the polynomial, so no conversion needs to be done, given that
@@ -93,6 +104,11 @@ func (b *Blob) toEncodedPayload(payloadForm codecs.PolynomialForm) (*encodedPayl
 		return nil, fmt.Errorf("invalid polynomial form")
 	}
 
+	maxPermissiblePayloadLength, err := codec.GetMaxPermissiblePayloadLength(b.blobLengthSymbols)
+	if err != nil {
+		return nil, fmt.Errorf("get max permissible payload length: %w", err)
+	}
+
 	encodedPayload, err := encodedPayloadFromElements(payloadElements, maxPermissiblePayloadLength)
 	if err != nil {
 		return nil, fmt.Errorf("encoded payload from elements %w", err)

api/clients/v2/codecs/blob_test.go → api/clients/v2/coretypes/blob_test.go

@@ -1,4 +1,4 @@
-package codecs
+package coretypes
 
 import (
 	"bytes"

api/clients/v2/codecs/encoded_payload.go → api/clients/v2/coretypes/encoded_payload.go

@@ -1,4 +1,4 @@
-package codecs
+package coretypes
 
 import (
 	"encoding/binary"

api/clients/v2/codecs/encoded_payload_test.go → api/clients/v2/coretypes/encoded_payload_test.go

@@ -1,4 +1,4 @@
-package codecs
+package coretypes
 
 import (
 	"testing"

api/clients/v2/codecs/payload.go → api/clients/v2/coretypes/payload.go

@@ -1,4 +1,4 @@
-package codecs
+package coretypes
 
 import (
 	"fmt"

api/clients/v2/payload_disperser.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/Layr-Labs/eigenda/api/clients/codecs"
+	"github.com/Layr-Labs/eigenda/api/clients/v2/coretypes"
 	"github.com/Layr-Labs/eigenda/api/clients/v2/verification"
 	dispgrpc "github.com/Layr-Labs/eigenda/api/grpc/disperser/v2"
 	"github.com/Layr-Labs/eigenda/common/geth"
@@ -24,7 +24,6 @@ import (
 type PayloadDisperser struct {
 	logger               logging.Logger
 	config               PayloadDisperserConfig
-	codec                codecs.BlobCodec
 	disperserClient      DisperserClient
 	certVerifier         verification.ICertVerifier
 	requiredQuorumsStore *RequiredQuorumsStore
@@ -87,20 +86,13 @@ func BuildPayloadDisperser(log logging.Logger, payloadDispCfg PayloadDisperserCo
 		return nil, fmt.Errorf("new cert verifier: %w", err)
 	}
 
-	// 5 - create codec
-	codec, err := codecs.CreateCodec(payloadDispCfg.PayloadPolynomialForm, payloadDispCfg.PayloadEncodingVersion)
-	if err != nil {
-		return nil, err
-	}
-
-	return NewPayloadDisperser(log, payloadDispCfg, codec, disperserClient, certVerifier)
+	return NewPayloadDisperser(log, payloadDispCfg, disperserClient, certVerifier)
 }
 
 // NewPayloadDisperser creates a PayloadDisperser from subcomponents that have already been constructed and initialized.
 func NewPayloadDisperser(
 	logger logging.Logger,
 	payloadDisperserConfig PayloadDisperserConfig,
-	codec codecs.BlobCodec,
 	// IMPORTANT: it is permissible for the disperserClient to be configured without a prover, but operating with this
 	// configuration puts a trust assumption on the disperser. With a nil prover, the disperser is responsible for computing
 	// the commitments to a blob, and the PayloadDisperser doesn't have a mechanism to verify these commitments.
@@ -125,7 +117,6 @@ func NewPayloadDisperser(
 	return &PayloadDisperser{
 		logger:               logger,
 		config:               payloadDisperserConfig,
-		codec:                codec,
 		disperserClient:      disperserClient,
 		certVerifier:         certVerifier,
 		requiredQuorumsStore: requiredQuorumsStore,
@@ -144,14 +135,12 @@ func (pd *PayloadDisperser) SendPayload(
 	ctx context.Context,
 	certVerifierAddress string,
 	// payload is the raw data to be stored on eigenDA
-	payload []byte,
+	payload *coretypes.Payload,
 ) (*verification.EigenDACert, error) {
-
-	blobBytes, err := pd.codec.EncodeBlob(payload)
+	blob, err := payload.ToBlob(pd.config.PayloadPolynomialForm)
 	if err != nil {
-		return nil, fmt.Errorf("encode payload to blob: %w", err)
+		return nil, fmt.Errorf("convert payload to blob: %w", err)
 	}
-	pd.logger.Debug("Payload encoded to blob")
 
 	timeoutCtx, cancel := context.WithTimeout(ctx, pd.config.ContractCallTimeout)
 	defer cancel()
@@ -162,9 +151,14 @@ func (pd *PayloadDisperser) SendPayload(
 
 	timeoutCtx, cancel = context.WithTimeout(ctx, pd.config.DisperseBlobTimeout)
 	defer cancel()
+
+	// TODO (litt3): eventually, we should consider making DisperseBlob accept an actual blob object, instead of the
+	//  serialized bytes. The operations taking place in DisperseBlob require the bytes to be converted into field
+	//  elements anyway, so serializing the blob here is unnecessary work. This will be a larger change that affects
+	//  many areas of code, though.
 	blobStatus, blobKey, err := pd.disperserClient.DisperseBlob(
 		timeoutCtx,
-		blobBytes,
+		blob.Serialize(),
 		pd.config.BlobVersion,
 		requiredQuorums,
 	)

api/clients/v2/payload_retriever.go

@@ -3,6 +3,7 @@ package clients
 import (
 	"context"
 
+	"github.com/Layr-Labs/eigenda/api/clients/v2/coretypes"
 	"github.com/Layr-Labs/eigenda/api/clients/v2/verification"
 )
 
@@ -12,5 +13,5 @@ import (
 // bucket instead of from EigenDA relays or nodes.
 type PayloadRetriever interface {
 	// GetPayload retrieves a payload from some backend, using the provided certificate
-	GetPayload(ctx context.Context, eigenDACert *verification.EigenDACert) ([]byte, error)
+	GetPayload(ctx context.Context, eigenDACert *verification.EigenDACert) (*coretypes.Payload, error)
 }

api/clients/v2/relay_payload_retriever.go

@@ -6,10 +6,9 @@ import (
 	"fmt"
 	"math/rand"
 
-	"github.com/Layr-Labs/eigenda/api/clients/codecs"
+	"github.com/Layr-Labs/eigenda/api/clients/v2/coretypes"
 	"github.com/Layr-Labs/eigenda/api/clients/v2/verification"
 	core "github.com/Layr-Labs/eigenda/core/v2"
-	"github.com/Layr-Labs/eigenda/encoding"
 	"github.com/Layr-Labs/eigensdk-go/logging"
 	"github.com/consensys/gnark-crypto/ecc/bn254"
 )
@@ -24,7 +23,6 @@ type RelayPayloadRetriever struct {
 	// must be evaluated for thread safety.
 	random      *rand.Rand
 	config      RelayPayloadRetrieverConfig
-	codec       codecs.BlobCodec
 	relayClient RelayClient
 	g1Srs       []bn254.G1Affine
 }
@@ -48,19 +46,11 @@ func BuildRelayPayloadRetriever(
 		return nil, fmt.Errorf("new relay client: %w", err)
 	}
 
-	codec, err := codecs.CreateCodec(
-		relayPayloadRetrieverConfig.PayloadPolynomialForm,
-		relayPayloadRetrieverConfig.PayloadEncodingVersion)
-	if err != nil {
-		return nil, err
-	}
-
 	return NewRelayPayloadRetriever(
 		log,
 		rand.New(rand.NewSource(rand.Int63())),
 		relayPayloadRetrieverConfig,
 		relayClient,
-		codec,
 		g1Srs)
 }
 
@@ -71,7 +61,6 @@ func NewRelayPayloadRetriever(
 	random *rand.Rand,
 	relayPayloadRetrieverConfig RelayPayloadRetrieverConfig,
 	relayClient RelayClient,
-	codec codecs.BlobCodec,
 	g1Srs []bn254.G1Affine) (*RelayPayloadRetriever, error) {
 
 	err := relayPayloadRetrieverConfig.checkAndSetDefaults()
@@ -83,7 +72,6 @@ func NewRelayPayloadRetriever(
 		log:         log,
 		random:      random,
 		config:      relayPayloadRetrieverConfig,
-		codec:       codec,
 		relayClient: relayClient,
 		g1Srs:       g1Srs,
 	}, nil
@@ -100,7 +88,7 @@ func NewRelayPayloadRetriever(
 // verified prior to calling this method.
 func (pr *RelayPayloadRetriever) GetPayload(
 	ctx context.Context,
-	eigenDACert *verification.EigenDACert) ([]byte, error) {
+	eigenDACert *verification.EigenDACert) (*coretypes.Payload, error) {
 
 	blobKey, err := eigenDACert.ComputeBlobKey()
 	if err != nil {
@@ -130,7 +118,9 @@ func (pr *RelayPayloadRetriever) GetPayload(
 	for _, val := range indices {
 		relayKey := relayKeys[val]
 
-		blob, err := pr.getBlobWithTimeout(ctx, relayKey, blobKey)
+		blobLengthSymbols := eigenDACert.BlobInclusionInfo.BlobCertificate.BlobHeader.Commitment.Length
+
+		blob, err := pr.retrieveBlobWithTimeout(ctx, relayKey, blobKey, blobLengthSymbols)
 		// if GetBlob returned an error, try calling a different relay
 		if err != nil {
 			pr.log.Warn(
@@ -141,17 +131,14 @@ func (pr *RelayPayloadRetriever) GetPayload(
 			continue
 		}
 
-		if uint(len(blob)) > blobCommitments.Length*encoding.BYTES_PER_SYMBOL {
-			pr.log.Warn(
-				"received length is greater than claimed blob length",
-				"blobKey", blobKey.Hex(),
-				"relayKey", relayKey,
-				"receivedLengthBytes", len(blob),
-				"claimedLengthBytes", blobCommitments.Length*encoding.BYTES_PER_SYMBOL)
-			continue
-		}
-
-		valid, err := verification.GenerateAndCompareBlobCommitment(pr.g1Srs, blob, blobCommitments.Commitment)
+		// TODO (litt3): eventually, we should make GenerateAndCompareBlobCommitment accept a blob, instead of the
+		//  serialization of a blob. Commitment generation operates on field elements, which is how a blob is stored
+		//  under the hood, so it's actually duplicating work to serialize the blob here. I'm declining to make this
+		//  change now, to limit the size of the refactor PR.
+		valid, err := verification.GenerateAndCompareBlobCommitment(
+			pr.g1Srs,
+			blob.Serialize(),
+			blobCommitments.Commitment)
 		if err != nil {
 			pr.log.Warn(
 				"generate and compare blob commitment",
@@ -165,14 +152,13 @@ func (pr *RelayPayloadRetriever) GetPayload(
 			continue
 		}
 
-		payload, err := pr.codec.DecodeBlob(blob)
+		payload, err := blob.ToPayload(pr.config.PayloadPolynomialForm)
 		if err != nil {
 			pr.log.Error(
-				`Cert verification was successful, but decode blob failed!
-					This is likely a problem with the local blob codec configuration,
-					but could potentially indicate a maliciously generated blob certificate.
-					It should not be possible for an honestly generated certificate to verify
-					for an invalid blob!`,
+				`Commitment verification was successful, but conversion from blob to payload failed!
+					This is likely a problem with the local configuration, but could potentially indicate
+					malicious dispersed data. It should not be possible for a commitment to verify for an
+					invalid blob!`,
 				"blobKey", blobKey.Hex(), "relayKey", relayKey, "eigenDACert", eigenDACert, "error", err)
 			return nil, fmt.Errorf("decode blob: %w", err)
 		}
@@ -183,16 +169,30 @@ func (pr *RelayPayloadRetriever) GetPayload(
 	return nil, fmt.Errorf("unable to retrieve blob %v from any relay. relay count: %d", blobKey.Hex(), relayKeyCount)
 }
 
-// getBlobWithTimeout attempts to get a blob from a given relay, and times out based on config.FetchTimeout
-func (pr *RelayPayloadRetriever) getBlobWithTimeout(
+// retrieveBlobWithTimeout attempts to retrieve a blob from a given relay, and times out based on config.FetchTimeout
+func (pr *RelayPayloadRetriever) retrieveBlobWithTimeout(
 	ctx context.Context,
 	relayKey core.RelayKey,
-	blobKey *core.BlobKey) ([]byte, error) {
+	blobKey *core.BlobKey,
+	// blobLengthSymbols should be taken from the eigenDACert for the blob being retrieved
+	blobLengthSymbols uint32,
+) (*coretypes.Blob, error) {
 
 	timeoutCtx, cancel := context.WithTimeout(ctx, pr.config.RelayTimeout)
 	defer cancel()
 
-	return pr.relayClient.GetBlob(timeoutCtx, relayKey, *blobKey)
+	// TODO (litt3): eventually, we should make GetBlob return an actual blob object, instead of the serialized bytes.
+	blobBytes, err := pr.relayClient.GetBlob(timeoutCtx, relayKey, *blobKey)
+	if err != nil {
+		return nil, fmt.Errorf("get blob from relay: %w", err)
+	}
+
+	blob, err := coretypes.DeserializeBlob(blobBytes, blobLengthSymbols)
+	if err != nil {
+		return nil, fmt.Errorf("deserialize blob: %w", err)
+	}
+
+	return blob, nil
 }
 
 // Close is responsible for calling close on all internal clients. This method will do its best to close all internal