Merge branch 'main' into renjie/refactor_exchagne_writer

Makefile.toml

@@ -700,7 +700,7 @@ script = """
 #!/usr/bin/env bash
 set -e
 
-cargo check -p risingwave_simulation "$@"
+cargo check -p risingwave_simulation --all-targets "$@"
 """
 
 [tasks.sslt]

ci/scripts/s3-source-test.sh

@@ -4,11 +4,14 @@ set -euo pipefail
 
 source ci/scripts/common.env.sh
 
-while getopts 'p:' opt; do
+while getopts 'p:s:' opt; do
     case ${opt} in
         p )
             profile=$OPTARG
             ;;
+        s )
+            script=$OPTARG
+            ;;
         \? )
             echo "Invalid Option: -$OPTARG" 1>&2
             exit 1
@@ -20,6 +23,8 @@ while getopts 'p:' opt; do
 done
 shift $((OPTIND -1))
 
+
+
 echo "--- Download artifacts"
 mkdir -p target/debug
 buildkite-agent artifact download risingwave-"$profile" target/debug/
@@ -44,7 +49,7 @@ cargo make ci-start ci-1cn-1fe
 
 echo "--- Run test"
 python3 -m pip install minio psycopg2-binary
-python3 e2e_test/s3/run.py
+python3 e2e_test/s3/$script.py
 
 echo "--- Kill cluster"
 cargo make ci-kill

ci/workflows/main-cron.yml

@@ -130,15 +130,15 @@ steps:
     retry: *auto-retry
 
   - label: "scaling test (deterministic simulation)"
-    command: "TEST_NUM=60 timeout 110m ci/scripts/deterministic-scale-test.sh"
+    command: "TEST_NUM=60 timeout 70m ci/scripts/deterministic-scale-test.sh"
     depends_on: "build-simulation"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v4.9.0:
           run: rw-build-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
-    timeout_in_minutes: 120
+    timeout_in_minutes: 70
     retry: *auto-retry
 
   - label: "end-to-end test (deterministic simulation)"
@@ -184,8 +184,8 @@ steps:
     timeout_in_minutes: 5
     retry: *auto-retry
 
-  - label: "S3 source check on AWS"
-    command: "ci/scripts/s3-source-test.sh -p ci-release"
+  - label: "S3 source check on AWS (json parser)"
+    command: "ci/scripts/s3-source-test.sh -p ci-release -s run"
     depends_on: build
     plugins:
       - seek-oss/aws-sm#v2.3.1:
@@ -200,8 +200,8 @@ steps:
     timeout_in_minutes: 20
     retry: *auto-retry  
 
-  - label: "S3 source check on lyvecloud.seagate.com"
-    command: "ci/scripts/s3-source-test.sh -p ci-release"
+  - label: "S3 source check on lyvecloud.seagate.com (json parser)"
+    command: "ci/scripts/s3-source-test.sh -p ci-release -s run"
     depends_on: build
     plugins:
       - seek-oss/aws-sm#v2.3.1:
@@ -215,3 +215,18 @@ steps:
             - S3_SOURCE_TEST_CONF
     timeout_in_minutes: 20
     retry: *auto-retry  
+  - label: "S3 source check on AWS (csv parser)"
+    command: "ci/scripts/s3-source-test.sh -p ci-release -s run_csv"
+    depends_on: build
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            S3_SOURCE_TEST_CONF: ci_s3_source_test_aws
+      - docker-compose#v4.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - S3_SOURCE_TEST_CONF
+    timeout_in_minutes: 20
+    retry: *auto-retry  

ci/workflows/main.yml

@@ -228,15 +228,15 @@ steps:
     retry: *auto-retry
 
   - label: "scaling test (deterministic simulation)"
-    command: "TEST_NUM=30 timeout 50m ci/scripts/deterministic-scale-test.sh"
+    command: "TEST_NUM=30 timeout 25m ci/scripts/deterministic-scale-test.sh"
     depends_on: "build-simulation"
     plugins:
       - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v4.9.0:
           run: rw-build-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
-    timeout_in_minutes: 55
+    timeout_in_minutes: 25
     retry: *auto-retry
 
   - label: "end-to-end test (deterministic simulation)"