feat: Use S3 node store with garage

.env

@@ -22,3 +22,7 @@ HEALTHCHECK_RETRIES=10
 POSTGRES_MAX_CONNECTIONS=100
 # Set SETUP_JS_SDK_ASSETS to 1 to enable the setup of JS SDK assets
 # SETUP_JS_SDK_ASSETS=1
+# Storage size for node store in Garage
+# If you need to change this after first install, you'd need to change the Garage layout
+# See https://garagehq.deuxfleurs.fr/documentation/quick-start/#creating-a-cluster-layout
+GARAGE_STORAGE_SIZE=100G

docker-compose.yml

@@ -34,6 +34,8 @@ x-sentry-defaults: &sentry_defaults
       <<: *depends_on-default
     smtp:
       <<: *depends_on-default
+    garage:
+      <<: *depends_on-default
     snuba-api:
       <<: *depends_on-default
     symbolicator:
@@ -139,6 +141,7 @@ services:
   kafka:
     <<: *restart_policy
     image: "confluentinc/cp-kafka:7.6.1"
+    user: root
     environment:
       # https://docs.confluent.io/platform/current/installation/docker/config-reference.html#cp-kakfa-example
       KAFKA_PROCESS_ROLES: "broker,controller"
@@ -207,6 +210,19 @@ services:
       interval: 10s
       timeout: 10s
       retries: 30
+  garage:
+    image: dxflrs/garage:v1.0.1
+    volumes:
+      - type: bind
+        read_only: true
+        source: ./garage.toml
+        target: /etc/garage.toml
+      - "sentry-garage:/var/lib/garage/"
+    healthcheck:
+      test: ["CMD", "/garage", "status"]
+      interval: 10s
+      timeout: 2s
+      retries: 2
   snuba-api:
     <<: *snuba_defaults
   # Kafka consumer responsible for feeding events into Clickhouse
@@ -540,3 +556,4 @@ volumes:
   sentry-kafka-log:
   sentry-smtp-log:
   sentry-clickhouse-log:
+  sentry-garage:

garage.toml

@@ -0,0 +1,19 @@
+metadata_dir = "/var/lib/garage/meta"
+data_dir = "/var/lib/garage/data"
+db_engine = "sqlite"
+
+replication_factor = 1
+# idk why rcp_ is needed without replication but ok
+rpc_bind_addr = "[::]:3901"
+rpc_public_addr = "0.0.0.0:3901"
+# This secret will not ever be used as we are in a single node setup
+# so this static value is OK. Don't forget to change it if you ever
+# decide to use garage in a multi-node setup.
+rpc_secret = "28f2fc651d4f280cea95e166f9113fe589039f9711d6c4f461872c1c5aaf7dec"
+
+# Setting 0 chooses the default which is 3 for now
+compression_level = 0
+
+[s3_api]
+s3_region = "garage"
+api_bind_addr = "[::]:3900"

install.sh

@@ -31,6 +31,7 @@ source install/ensure-relay-credentials.sh
 source install/generate-secret-key.sh
 source install/update-docker-images.sh
 source install/build-docker-images.sh
+source install/bootstrap-garage.sh
 source install/bootstrap-snuba.sh
 source install/upgrade-postgres.sh
 source install/set-up-and-migrate-database.sh

install/bootstrap-garage.sh

@@ -0,0 +1,37 @@
+echo "${_group}Bootstrapping garage (node store)..."
+
+$dc up --wait garage postgres
+garage="$dc exec garage /garage"
+
+if [[ $($garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; then
+  node_id=$($garage status | tail -1 | awk '{print $1}')
+  $garage layout assign -z local -c $GARAGE_STORAGE_SIZE "$node_id"
+  $garage layout apply --version 1
+
+  # Only touch if no existing nodestore config is found
+  if ! grep -q "SENTRY_NODESTORE" $SENTRY_CONFIG_PY; then
+    nodestore_config=$(sed -n '/SENTRY_NODESTORE/,/[}]/{p}' sentry/sentry.conf.example.py)
+    if [[ $($dc exec postgres psql -qAt -U postgres -c "select exists (select * from nodestore_node limit 1)") = "f" ]]; then
+      nodestore_config=$(echo -e "$nodestore_config" | sed '$s/\}/    "read_through": True,\n    "delete_through": True,\n\}/')
+    fi
+    echo "$nodestore_config" >>$SENTRY_CONFIG_PY
+  fi
+
+  $garage bucket create nodestore
+  key_info=$($garage key create nodestore-key | head -3 | tail -2)
+  echo "$key_info"
+  key_id=$(echo "$key_info" | head -1 | awk '{print $3}')
+  key_secret=$(echo "$key_info" | tail -1 | awk '{print $3}')
+
+  $garage bucket allow --read --write --owner nodestore --key nodestore-key
+
+  if grep -q "<GARAGE_KEY_ID>" $SENTRY_CONFIG_PY; then
+    sed -i -e "s/<GARAGE_KEY_ID>/$key_id/" $SENTRY_CONFIG_PY
+    sed -i -e "s/<GARAGE_SECRET_KEY>/$key_secret/" $SENTRY_CONFIG_PY
+    echo "Set Garage keys for SENTRY_NODESTORE_OPTIONS in $SENTRY_CONFIG_PY"
+  fi
+else
+  echo "Node store already exists, skipping..."
+fi
+
+echo "${_endgroup}"

sentry/Dockerfile

@@ -1,13 +1,15 @@
 ARG SENTRY_IMAGE
 FROM ${SENTRY_IMAGE}
 
+RUN pip install https://github.com/stayallive/sentry-nodestore-s3/archive/main.zip
+
 COPY . /usr/src/sentry
 
 RUN if [ -s /usr/src/sentry/enhance-image.sh ]; then \
     /usr/src/sentry/enhance-image.sh; \
-fi
+    fi
 
 RUN if [ -s /usr/src/sentry/requirements.txt ]; then \
     echo "sentry/requirements.txt is deprecated, use sentry/enhance-image.sh - see https://develop.sentry.dev/self-hosted/#enhance-sentry-image"; \
     pip install -r /usr/src/sentry/requirements.txt; \
-fi
+    fi

sentry/sentry.conf.example.py

@@ -84,6 +84,21 @@ def get_internal_network():
 # See https://develop.sentry.dev/self-hosted/experimental/errors-only/
 SENTRY_SELF_HOSTED_ERRORS_ONLY = env("COMPOSE_PROFILES") != "feature-complete"
 
+##############
+# Node Store #
+##############
+
+SENTRY_NODESTORE = "sentry_nodestore_s3.S3PassthroughDjangoNodeStorage"
+SENTRY_NODESTORE_OPTIONS = {
+    "compression": False,  # we have compression enabled in Garage itself
+    "endpoint_url": "http://garage:3900",
+    "bucket_path": "nodestore",
+    "bucket_name": "nodestore",
+    "region_name": "garage",
+    "aws_access_key_id": "<GARAGE_KEY_ID>",
+    "aws_secret_access_key": "<GARAGE_SECRET_KEY>",
+}
+
 #########
 # Redis #
 #########