Release/pectra (#399)

* feat: Add nginx server configuration for clickhouse * Initial electra * feat: Add support for Electra block in beacon block processing * chore: Remove xatu-cannon service from docker-compose.yml * feat: Add CANNON_NETWORK_NAME to docker-compose.yml * feat: Enhance logging for network name override * feat: Add support for Electra beacon block message * style: Add resolver configuration for nginx servers * feat: Add password configuration for ClickHouse users * feat: Add access control improvements and constraints * refactor: Update version of go-eth2-client to v0.21.12 * fix: Update version of go-eth2-client to v0.21.12-0.20241014214053-759b0ace2e43 * refactor: Update go-eth2-client version to v0.23.1 * build: Update module versions and add beacon v0.45.1 * refactor: Remove unnecessary slot variable and optimize epoch handling * refactor: Refactor event block creation in block.go * feat: Enable backfill iterator for beaconBlock and beaconBlobSidecar * style: remove unnecessary comment in xatu-cannon.yaml * Add ExecutionRequests to Electra beacon block * feat: update go dependencies and beacon node config This commit updates the go dependencies to their latest versions. It also adds a new configuration option to the beacon node to enforce JSON responses. This is done to ensure that the beacon node is only processing valid JSON data. * feat(beacon): enforce JSON content type for eth2 client requests
ethpandaops · Feb 10, 2025 · 4be34a4 · 4be34a4
1 parent d3ec12d
commit 4be34a4
Show file tree

Hide file tree

Showing 27 changed files with 3,118 additions and 1,181 deletions.
diff --git a/...y/local/docker-compose/clickhouse/clickhouse-01/etc/clickhouse-server/config.d/config.xml b/...y/local/docker-compose/clickhouse/clickhouse-01/etc/clickhouse-server/config.d/config.xml
@@ -55,4 +55,7 @@
         <shard>01</shard>
         <replica>01</replica>
     </macros>
+    <access_control_improvements>
+        <settings_constraints_replace_previous>true</settings_constraints_replace_previous>
+    </access_control_improvements>
 </clickhouse>
diff --git a/deploy/local/docker-compose/clickhouse/clickhouse-01/etc/clickhouse-server/users.d/users.xml b/deploy/local/docker-compose/clickhouse/clickhouse-01/etc/clickhouse-server/users.d/users.xml
@@ -11,8 +11,15 @@
             <max_memory_usage>10000000000</max_memory_usage>
             <use_uncompressed_cache>0</use_uncompressed_cache>
             <load_balancing>in_order</load_balancing>
-            <readonly>1</readonly>
+            <readonly>2</readonly>
             <log_queries>1</log_queries>
+            <constraints>
+                <max_execution_time>
+                <changeable_in_readonly/>
+                <min>0</min>
+                <max>180</max>
+                </max_execution_time>
+            </constraints>
         </readonly>
     </profiles>
     <users>
@@ -24,6 +31,7 @@
             </networks>
             <quota>default</quota>
             <access_management>1</access_management>
+            <password from_env="CLICKHOUSE_PASSWORD" replace="replace"></password>
             <named_collection_control>1</named_collection_control>
             <show_named_collections>1</show_named_collections>
             <show_named_collections_secrets>1</show_named_collections_secrets>

diff --git a/...y/local/docker-compose/clickhouse/clickhouse-02/etc/clickhouse-server/config.d/config.xml b/...y/local/docker-compose/clickhouse/clickhouse-02/etc/clickhouse-server/config.d/config.xml
@@ -55,4 +55,7 @@
         <shard>02</shard>
         <replica>01</replica>
     </macros>
+    <access_control_improvements>
+        <settings_constraints_replace_previous>true</settings_constraints_replace_previous>
+    </access_control_improvements>
 </clickhouse>
diff --git a/deploy/local/docker-compose/clickhouse/clickhouse-02/etc/clickhouse-server/users.d/users.xml b/deploy/local/docker-compose/clickhouse/clickhouse-02/etc/clickhouse-server/users.d/users.xml
@@ -11,8 +11,15 @@
             <max_memory_usage>10000000000</max_memory_usage>
             <use_uncompressed_cache>0</use_uncompressed_cache>
             <load_balancing>in_order</load_balancing>
-            <readonly>1</readonly>
+            <readonly>2</readonly>
             <log_queries>1</log_queries>
+            <constraints>
+                <max_execution_time>
+                <changeable_in_readonly/>
+                <min>0</min>
+                <max>180</max>
+                </max_execution_time>
+            </constraints>
         </readonly>
     </profiles>
     <users>
@@ -22,6 +29,7 @@
             <networks>
                 <ip>::/0</ip>
             </networks>
+            <password from_env="CLICKHOUSE_PASSWORD" replace="replace"></password>
             <quota>default</quota>
             <access_management>1</access_management>
             <named_collection_control>1</named_collection_control>

diff --git a/deploy/local/docker-compose/nginx/nginx.conf b/deploy/local/docker-compose/nginx/nginx.conf
@@ -30,7 +30,8 @@ http {
     server {
         listen 80;
         server_name grafana.$HOSTNAME;
-
+        resolver 127.0.0.11 valid=10s;
+        resolver_timeout 5s; 
         location / {
             proxy_pass http://xatu-grafana:3000;
             proxy_set_header Host $host;
@@ -43,7 +44,8 @@ http {
     server {
         listen 80 http2;
         server_name server.$HOSTNAME;
-
+        resolver 127.0.0.11 valid=10s;
+        resolver_timeout 5s; 
         location / {
             grpc_pass grpc://xatu-server:8080;
             proxy_set_header Host $host;
@@ -52,4 +54,18 @@ http {
             proxy_set_header X-Forwarded-Proto $scheme;
         }
     }
+
+    server {
+        listen 80;
+        server_name clickhouse.$HOSTNAME;
+        resolver 127.0.0.11 valid=10s;
+        resolver_timeout 5s; 
+        location / {
+            proxy_pass http://xatu-clickhouse-01:8123;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+        }
+    }
 }
diff --git a/deploy/local/docker-compose/xatu-cannon.yaml b/deploy/local/docker-compose/xatu-cannon.yaml
@@ -4,7 +4,7 @@ pprofAddr: ":6061" # optional. if supplied it enables pprof server
 
 name: xatu-cannon
 
-# derivers:
+derivers:
 #   attesterSlashing:
 #     enabled: false
 #   blsToExecutionChange:
@@ -19,10 +19,17 @@ name: xatu-cannon
 #     enabled: false
 #   voluntaryExit:
 #     enabled: false
-#   beaconBlock:
-#     enabled: true
-#   beaconBlobSidecar:
-#     enabled: false
+  beaconBlock:
+    enabled: true
+    iterator:
+      backfill:
+        enabled: true
+  beaconBlobSidecar:
+    enabled: true
+    iterator:
+      backfill:
+        enabled: true
+
 #   proposerDuty:
 #     enabled: true
 #   elaboratedAttestation:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -376,7 +376,7 @@ services:
     environment:
       KAFKA_BROKERS: "xatu-kafka:29092"
     ports:
-      - "${VECTOR_HTTP_KAFKA_ADDRESS:-127.0.0.1}:${VECTOR_HTTP_KAFKA_PORT:-9005}:9005"
+      - "${VECTOR_HTTP_KAFKA_ADDRESS:-127.0.0.1}:${VECTOR_HTTP_KAFKA_PORT:-9055}:9005"
     networks:
       - xatu-net
     healthcheck:
@@ -563,6 +563,7 @@ services:
       CANNON_XATU_OUTPUT_AUTHORIZATION: ${CANNON_XATU_OUTPUT_AUTHORIZATION:-Basic eGF0dTpleGFtcGxl}
       # Default of http://localhost:5052
       CANNON_BEACON_NODE_URL: ${CANNON_BEACON_NODE_URL:-http://localhost:5052}
+      CANNON_NETWORK_NAME: ${CANNON_NETWORK_NAME}
     volumes:
       - ./deploy/local/docker-compose/xatu-cannon.yaml:/etc/cannon/config.yaml
     networks:

diff --git a/go.mod b/go.mod
@@ -8,24 +8,24 @@ replace github.com/prysmaticlabs/prysm/v5 => github.com/ethpandaops/prysm/v5 v5.
 
 require (
 	github.com/IBM/sarama v1.43.0
-	github.com/attestantio/go-eth2-client v0.21.9
+	github.com/attestantio/go-eth2-client v0.24.0
 	github.com/avast/retry-go/v4 v4.5.1
 	github.com/beevik/ntp v1.3.1
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9
 	github.com/creasty/defaults v1.7.0
-	github.com/ethereum/go-ethereum v1.14.10
-	github.com/ethpandaops/beacon v0.45.0
+	github.com/ethereum/go-ethereum v1.15.0
+	github.com/ethpandaops/beacon v0.47.0
 	github.com/ethpandaops/ethcore v0.0.0-20240422023000-2a5727b18756
 	github.com/ethpandaops/ethwallclock v0.3.0
-	github.com/ferranbt/fastssz v0.1.3
+	github.com/ferranbt/fastssz v0.1.4
 	github.com/go-co-op/gocron v1.27.1
 	github.com/golang/protobuf v1.5.4
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb
 	github.com/google/uuid v1.6.0
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
-	github.com/holiman/uint256 v1.3.1
+	github.com/holiman/uint256 v1.3.2
 	github.com/huandu/go-sqlbuilder v1.25.0
 	github.com/jellydator/ttlcache/v3 v3.2.0
 	github.com/klauspost/compress v1.17.9
@@ -41,14 +41,15 @@ require (
 	github.com/prysmaticlabs/prysm/v5 v5.1.0
 	github.com/redis/go-redis/v9 v9.5.1
 	github.com/sirupsen/logrus v1.9.3
-	github.com/spf13/cobra v1.8.0
+	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.9.0
 	go.opentelemetry.io/otel v1.29.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0
 	go.opentelemetry.io/otel/sdk v1.29.0
 	go.opentelemetry.io/otel/trace v1.29.0
-	golang.org/x/sync v0.8.0
+	golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa
+	golang.org/x/sync v0.10.0
 	google.golang.org/grpc v1.67.0
 	google.golang.org/protobuf v1.34.2
 	gopkg.in/yaml.v2 v2.4.0
@@ -57,7 +58,6 @@ require (
 
 require (
 	github.com/BurntSushi/toml v1.3.2 // indirect
-	github.com/DataDog/zstd v1.5.5 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/VictoriaMetrics/fastcache v1.12.2 // indirect
 	github.com/aristanetworks/goarista v0.0.0-20200805130819-fd197cf57d96 // indirect
@@ -69,24 +69,18 @@ require (
 	github.com/aws/smithy-go v1.20.1 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bits-and-blooms/bitset v1.13.0 // indirect
+	github.com/bits-and-blooms/bitset v1.17.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/cockroachdb/errors v1.11.3 // indirect
-	github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
-	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
-	github.com/cockroachdb/pebble v1.1.2 // indirect
-	github.com/cockroachdb/redact v1.1.5 // indirect
-	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
-	github.com/consensys/bavard v0.1.13 // indirect
-	github.com/consensys/gnark-crypto v0.12.1 // indirect
+	github.com/consensys/bavard v0.1.22 // indirect
+	github.com/consensys/gnark-crypto v0.14.0 // indirect
 	github.com/containerd/cgroups v1.1.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
-	github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect
-	github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
-	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect
+	github.com/crate-crypto/go-kzg-4844 v1.1.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/deckarep/golang-set/v2 v2.6.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
@@ -99,18 +93,18 @@ require (
 	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/elastic/gosigar v0.14.3 // indirect
+	github.com/emicklei/dot v1.6.4 // indirect
 	github.com/ethereum/c-kzg-4844 v1.0.0 // indirect
-	github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 // indirect
-	github.com/fatih/color v1.16.0 // indirect
+	github.com/ethereum/go-verkle v0.2.2 // indirect
+	github.com/fatih/color v1.18.0 // indirect
 	github.com/flynn/noise v1.1.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/getsentry/sentry-go v0.27.0 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
-	github.com/goccy/go-yaml v1.9.8 // indirect
+	github.com/goccy/go-yaml v1.9.5 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -129,7 +123,7 @@ require (
 	github.com/herumi/bls-eth-go-binary v0.0.0-20210917013441-d37c07cfda4e // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
 	github.com/huandu/go-clone v1.6.0 // indirect
-	github.com/huandu/xstrings v1.4.0 // indirect
+	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/huin/goupnp v1.3.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/ipfs/go-cid v0.4.1 // indirect
@@ -144,7 +138,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
 	github.com/koron/go-ssdp v0.0.4 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
@@ -161,7 +155,7 @@ require (
 	github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
 	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
@@ -206,16 +200,18 @@ require (
 	github.com/pion/sdp/v3 v3.0.9 // indirect
 	github.com/pion/srtp/v2 v2.0.20 // indirect
 	github.com/pion/stun v0.6.1 // indirect
+	github.com/pion/stun/v2 v2.0.0 // indirect
 	github.com/pion/transport/v2 v2.2.10 // indirect
+	github.com/pion/transport/v3 v3.0.7 // indirect
 	github.com/pion/turn/v2 v2.1.6 // indirect
 	github.com/pion/webrtc/v3 v3.3.0 // indirect
-	github.com/pk910/dynamic-ssz v0.0.3 // indirect
-	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/pk910/dynamic-ssz v0.0.4 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/prometheus/prom2json v1.3.0 // indirect
 	github.com/prysmaticlabs/fastssz v0.0.0-20241008181541-518c4ce73516 // indirect
-	github.com/prysmaticlabs/go-bitfield v0.0.0-20240328144219-a1caa50c3a1e // indirect
+	github.com/prysmaticlabs/go-bitfield v0.0.0-20240618144021-706c95b2dd15 // indirect
 	github.com/prysmaticlabs/gohashtree v0.0.4-beta.0.20240624100937-73632381301b // indirect
 	github.com/prysmaticlabs/prombbolt v0.0.0-20210126082820-9b7adba6db7c // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
@@ -232,8 +228,8 @@ require (
 	github.com/schollz/progressbar/v3 v3.3.4 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/spf13/afero v1.11.0 // indirect
-	github.com/spf13/cast v1.6.0 // indirect
+	github.com/spf13/afero v1.10.0 // indirect
+	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/supranational/blst v0.3.13 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
@@ -260,17 +256,16 @@ require (
 	go.uber.org/mock v0.4.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/crypto v0.26.0 // indirect
-	golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
-	golang.org/x/mod v0.20.0 // indirect
-	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/crypto v0.32.0 // indirect
+	golang.org/x/mod v0.22.0 // indirect
+	golang.org/x/net v0.34.0 // indirect
 	golang.org/x/oauth2 v0.22.0 // indirect
-	golang.org/x/sys v0.24.0 // indirect
-	golang.org/x/term v0.23.0 // indirect
-	golang.org/x/text v0.17.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/term v0.28.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.24.0 // indirect
-	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+	golang.org/x/tools v0.29.0 // indirect
+	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd // indirect
 	gopkg.in/Knetic/govaluate.v3 v3.0.0 // indirect