Merge pull request #26 from rubytogether/segiddins/clickhouse

Add tests + functionality for streaming to clickhouse

Author: segiddins

Cargo.lock

@@ -10,20 +10,25 @@ default-run = "kirby"
 argparse = "0.2.2"
 aws_lambda_events = "0.16.0"
 enum-map = { version = "0.4.1", features = ["serde"] }
+expect-test = "1.5.1"
 flate2 = { version = "1.0", features = ["zlib-rs"], default-features = false }
 lambda_runtime = { version = "0.13.0", features = ["tracing"] }
 lazy_static = "1.1.0"
 log = "0.4.5"
 percent-encoding = "2.1.0"
 rayon = "1.0.2"
-regex = "1"
-rusoto_core = { version="0.48", default-features=false, features=["rustls"] }
-rusoto_s3 = { version="0.48", default-features=false, features=["rustls"] }
+regex = "^1.11"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
+serde_with = { version = "3.12.0", default-features = false, features = [
+  "macros",
+] }
 time = "0.1"
 tokio = "1.44.1"
+aws-config = { version = "1.6.0", features = ["behavior-version-latest"] }
+aws-sdk-s3 = { version = "1.79.0", features = ["rt-tokio"] }
+aws-credential-types = "1.2.2"
 
 [profile.release]
 debug = false

Cargo.toml

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -euo pipefail
 
-command -v hyperfine > /dev/null || (echo "You need to install 'hyperfine' with cargo or brew." && exit 1)
+command -v hyperfine >/dev/null || (echo "You need to install 'hyperfine' with cargo or brew." && exit 1)
 
 if [[ -z "${1:-}" ]]; then
   mkdir -p logs
@@ -32,7 +32,7 @@ if [[ -n "${PGO:-}" ]]; then
 
   # STEP 4: Use the `.profdata` file for guiding optimizations
   RUSTFLAGS="-Cprofile-use=/tmp/pgo-data/merged.profdata" \
-      cargo build --release --bin kirby
+    cargo build --release --bin kirby
 else
   cargo build --bin kirby --release
 fi

bin/bench

@@ -1,17 +1,23 @@
 #!/bin/bash
 set -euo pipefail
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 "$DIR/build-lambda"
 echo "Built new lambda binary"
 
 set -x
 cd target/x86_64-unknown-linux-musl/release
 zip aws_lambda.zip kirby-s3
+cp kirby-s3-clickhouse bootstrap
+zip aws_lambda_clickhouse.zip bootstrap
+rm bootstrap
 cd -
 
 aws lambda update-function-code \
   --function-name "kirby-s3" \
   --zip-file "fileb://target/x86_64-unknown-linux-musl/release/aws_lambda.zip"
+aws lambda update-function-code \
+  --function-name "kirby-s3-clickhouse" \
+  --zip-file "fileb://target/x86_64-unknown-linux-musl/release/aws_lambda_clickhouse.zip"
 set +x
 echo "Deployed new lambda binary"

bin/deploy-lambda

@@ -0,0 +1,63 @@
+## Request Line
+
+field|type|description
+---|---|---
+timestamp|DateTime|Timestamp of the event
+request_path|String|Path of the request
+request_query|String|Query string of the request
+user_agent|UserAgent|User agent of the client
+tls_cipher|String|Cipher used for the connection
+time_elapsed|u32|Time elapsed for the request
+client_continent|String|Continent of the client
+client_country|String|Country of the client
+client_region|String|Region of the client
+client_city|String|City of the client
+client_latitude|nullable String|Latitude of the client
+client_longitude|nullable String|Longitude of the client
+client_timezone|String|Timezone of the client
+client_connection|String|Connection type of the client
+request|String|Request method
+request_host|String|Host of the request
+request_bytes|u32|Bytes sent in the request
+http2|bool|HTTP/2 support
+tls|nullable bool|whether the connection was encrypted
+tls_version|String|TLS version used
+response_status|u32|HTTP status code
+response_text|String|HTTP status text
+response_bytes|u32|Bytes sent in the response
+response_cache|low cardinality String|Cache status of the response
+cache_state|low cardinality String|Cache state of the response
+cache_lastuse|f32|Time since the cache was last used
+cache_hits|u32|Number of cache hits
+server_region|low cardinality String|Region of the server
+server_datacenter|low cardinality String|Datacenter of the server
+gem|String|Name of the downloaded gem
+version|String|Version of the downloaded gem
+platform|Platform|Platform of the downloaded gem
+
+## UserAgent
+
+null values will be omitted
+
+field|type|description
+---|---|---
+agent_name|nullable String|Name of the user agent
+agent_version|nullable String|Version of the user agent
+bundler|nullable String|Bundler version
+rubygems|nullable String|Rubygems version
+ruby|nullable String|Ruby version
+platform|nullable Platform|Platform
+command|nullable String|Bundler command run
+options|nullable String|Options passed to the command
+jruby|nullable String|JRuby version
+truffleruby|nullable String|TruffleRuby version
+ci|nullable String|CI system(s), comma separated
+gemstash|nullable String|Gemstash version
+
+## Platform
+
+field|type|description
+---|---|---
+cpu|low cardinality nullable String|CPU architecture
+os|low cardinality nullable String|Operating system
+os_version|nullable String|Operating system version