Bundle C++ to SDK to artifact that can be used with CMake's fetch_con…

…tent (#3875) ### What * Fixes #3759 Change CMakeLists.txt in rerun_cpp to work standalone with add_directory. When used in this fashion it expects the `RERUN_C_LIB` variable to be set. The default for it points to a local `lib` folder with appropriately named libs. This in turn gives rise to a very trivial bundling strategy: * copy src * copy CMakeLists.txt * copy libs This is done in the new `build_and_upload_rerun_cpp` script which on top of the above also takes care of testing the resulting artifact by running the now also compatible minimal example outside of the repo. Another smaller necessary change is that the rerun.h C header is now copied into the rerun_cpp sdk with a patched version number in the comment. This is primarily so to keep the above process super simple and shave of an additional include directory. ~Draft until PR ci confirmed to be working.~ (PR checks only the linux artifact, main ci checks the full artifact and puts it on the Development Release notes) With this change, the minimal possible cmake script for a project using Rerun is: ```cmake project(hello_rerun) include(FetchContent) FetchContent_Declare(rerun_sdk URL https://build.rerun.io/commit/f4e5ad6/rerun_cpp.zip DOWNLOAD_EXTRACT_TIMESTAMP YES) FetchContent_MakeAvailable(rerun_sdk) add_executable(hello_rerun main.cpp) set_property(TARGET hello_rerun PROPERTY CXX_STANDARD 20) # Rerun requires C++17, but you can of course use an ever newer version. target_link_libraries(hello_rerun PRIVATE rerun_sdk) ``` ⚠️ **this STILL requires libarrow to be installed somewhere where cmake can find it** ⚠️ ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested [demo.rerun.io](https://demo.rerun.io/pr/3875) (if applicable) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG - [PR Build Summary](https://build.rerun.io/pr/3875) - [Docs preview](https://rerun.io/preview/9ddbc213c8f4bd2ea288590f9ab2f2b9d48ae552/docs)  - [Examples preview](https://rerun.io/preview/9ddbc213c8f4bd2ea288590f9ab2f2b9d48ae552/examples)  - [Recent benchmark results](https://ref.rerun.io/dev/bench/) - [Wasm size tracking](https://ref.rerun.io/dev/sizes/)
rerun-io · Oct 17, 2023 · 30dc8be · 30dc8be
1 parent 59fbbb8
commit 30dc8be
Show file tree

Hide file tree

Showing 15 changed files with 627 additions and 58 deletions.
diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml
@@ -44,6 +44,15 @@ jobs:
       UPLOAD_COMMIT_OVERRIDE: ${{ github.event.pull_request.head.sha }}
     secrets: inherit
 
+  bundle-and-upload-rerun_cpp:
+    name: "Bundle and upload rerun_cpp"
+    needs: [build-rerun_c-and-upload]
+    uses: ./.github/workflows/reusable_bundle_and_upload_rerun_cpp.yml
+    with:
+      PLATFORM_FILTER: "linux"
+      UPLOAD_COMMIT_OVERRIDE: ${{ github.event.pull_request.head.sha }}
+    secrets: inherit
+
   build-web:
     name: "Build Web"
     if: github.event.pull_request.head.repo.owner.login == 'rerun-io'
@@ -124,4 +133,3 @@ jobs:
       CONCURRENCY: pr-${{ github.event.pull_request.number }}
       PR_NUMBER: ${{ github.event.pull_request.number }}
     secrets: inherit
-
diff --git a/.github/workflows/on_push_main.yml b/.github/workflows/on_push_main.yml
@@ -227,6 +227,18 @@ jobs:
       CONCURRENCY: push-${{ github.ref_name }}
     secrets: inherit
 
+  bundle-and-upload-rerun_cpp:
+    name: "Bundle and upload rerun_cpp"
+    needs:
+      [
+        build-rerun_c-and-upload-linux,
+        build-rerun_c-and-upload-macos-intel,
+        build-rerun_c-and-upload-macos-arm,
+        build-rerun_c-and-upload-windows,
+      ]
+    uses: ./.github/workflows/reusable_bundle_and_upload_rerun_cpp.yml
+    secrets: inherit
+
   pre-release:
     name: Pre Release
     concurrency:
@@ -240,6 +252,7 @@ jobs:
         build-rerun_c-and-upload-macos-intel,
         build-rerun_c-and-upload-macos-arm,
         build-rerun_c-and-upload-windows,
+        bundle-and-upload-rerun_cpp,
       ]
     runs-on: "ubuntu-latest"
     steps:
@@ -273,7 +286,14 @@ jobs:
 
             ## Wheels can be installed with:
             ```
-            pip install --pre -f https://build.rerun.io/commit/${{ env.SHORT_SHA}}/wheels --upgrade rerun-sdk
+            pip install --pre -f https://build.rerun.io/commit/${{ env.SHORT_SHA }}/wheels --upgrade rerun-sdk
+            ```
+
+            ## CMake fetch-content for C++ SDK
+            ```
+            include(FetchContent)
+            FetchContent_Declare(rerun_sdk URL https://build.rerun.io/commit/${{ env.SHORT_SHA }}/rerun_cpp_sdk.zip)
+            FetchContent_MakeAvailable(rerun_sdk)
             ```
 
             ## Static libraries for rerun_c
@@ -290,4 +310,3 @@ jobs:
           allowUpdates: true
           removeArtifacts: true
           replacesArtifacts: true
-
diff --git a/.github/workflows/reusable_build_and_upload_rerun_c.yml b/.github/workflows/reusable_build_and_upload_rerun_c.yml
@@ -82,7 +82,6 @@ jobs:
     outputs:
       RUNNER: ${{ steps.set-config.outputs.runner }}
       TARGET: ${{ steps.set-config.outputs.target }}
-      RUN_TESTS: ${{ steps.set-config.outputs.run_tests }}
       CONTAINER: ${{ steps.set-config.outputs.container }}
       LIB_NAME: ${{ steps.set-config.outputs.lib_name }}
     steps:
@@ -93,36 +92,31 @@ jobs:
             linux)
               runner="ubuntu-latest-16-cores"
               target="x86_64-unknown-linux-gnu"
-              run_tests="true"
               container="{'image': 'rerunio/ci_docker:0.10.0'}"
               lib_name="librerun_c.a"
               ;;
             windows)
               runner="windows-latest-8-cores"
               target="x86_64-pc-windows-msvc"
-              run_tests="true"
               container="null"
               lib_name="rerun_c.lib"
               ;;
             macos-arm)
               runner="macos-latest"
               target="aarch64-apple-darwin"
-              run_tests="false"
               container="null"
               lib_name="librerun_c.a"
               ;;
             macos-intel)
               runner="macos-latest"
               target="x86_64-apple-darwin"
-              run_tests="false"
               container="null"
               lib_name="librerun_c.a"
               ;;
             *) echo "Invalid platform" && exit 1 ;;
           esac
           echo "runner=$runner" >> "$GITHUB_OUTPUT"
           echo "target=$target" >> "$GITHUB_OUTPUT"
-          echo "run_tests=$run_tests" >> "$GITHUB_OUTPUT"
           echo "container=$container" >> "$GITHUB_OUTPUT"
           echo "lib_name=$lib_name" >> "$GITHUB_OUTPUT"
 
@@ -163,7 +157,7 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: build
-          args: --locked -p rerun_c --release
+          args: --locked -p rerun_c --release --target ${{ needs.set-config.outputs.TARGET }}
 
       - id: "auth"
         uses: google-github-actions/auth@v1
@@ -182,19 +176,18 @@ jobs:
           fi
           echo "SHORT_SHA=$(echo $USED_SHA | cut -c1-7)" >> $GITHUB_OUTPUT
 
-      - name: "Upload web-viewer (commit)"
+      - name: "Upload rerun_c (commit)"
         if: ${{ inputs.UPLOAD_COMMIT }}
         uses: google-github-actions/upload-cloud-storage@v1
         with:
-          path: "./target/release/${{ needs.set-config.outputs.LIB_NAME }}"
+          path: "./target/${{ needs.set-config.outputs.TARGET }}/release/${{ needs.set-config.outputs.LIB_NAME }}"
           destination: "rerun-builds/commit/${{ steps.short-sha.outputs.SHORT_SHA }}/rerun_c/${{ inputs.PLATFORM }}"
           parent: false
 
-      - name: "Upload web-viewer (adhoc)"
+      - name: "Upload rerun_c (adhoc)"
         if: ${{ inputs.ADHOC_NAME != '' }}
         uses: google-github-actions/upload-cloud-storage@v1
         with:
-          path: "./target/release/${{ needs.set-config.outputs.LIB_NAME }}"
+          path: "./target/${{ needs.set-config.outputs.TARGET }}/release/${{ needs.set-config.outputs.LIB_NAME }}"
           destination: "rerun-builds/adhoc/${{inputs.ADHOC_NAME}}/rerun_c/${{ inputs.PLATFORM }}"
           parent: false
-
diff --git a/.github/workflows/reusable_bundle_and_upload_rerun_cpp.yml b/.github/workflows/reusable_bundle_and_upload_rerun_cpp.yml
@@ -0,0 +1,61 @@
+name: Reusable C++ bundling and upload
+
+on:
+  workflow_call:
+    inputs:
+      PLATFORM_FILTER:
+        required: false
+        type: string
+      UPLOAD_COMMIT_OVERRIDE:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  bundle-and-upload-rerun_cpp:
+    name: C++ bundling and upload
+
+    permissions:
+      contents: "read"
+      id-token: "write"
+      pull-requests: "write"
+
+    runs-on: ubuntu-latest
+    # Need container for arrow dependency.
+    container:
+      image: rerunio/ci_docker:0.10.0
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.sha }}
+
+      - id: "auth"
+        uses: google-github-actions/auth@v1
+        with:
+          workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
+
+      - name: "Set up Cloud SDK"
+        uses: "google-github-actions/setup-gcloud@v1"
+        with:
+          version: ">= 363.0.0"
+
+      - name: Install python gcs library
+        run: |
+          python3 -m pip install google-cloud-storage
+
+      - name: Add SHORT_SHA env property with commit short sha
+        shell: bash
+        id: "short-sha"
+        run: |
+          if [ -z "${{ inputs.UPLOAD_COMMIT_OVERRIDE }}" ]; then
+            USED_SHA=${{ github.sha }}
+          else
+            USED_SHA=${{ inputs.UPLOAD_COMMIT_OVERRIDE }}
+          fi
+          echo "SHORT_SHA=$(echo $USED_SHA | cut -c1-7)" >> $GITHUB_OUTPUT
+
+      - name: "Bundle & Upload rerun_cpp"
+        run: python3 ./scripts/ci/bundle_and_upload_rerun_cpp.py --git-hash ${{ steps.short-sha.outputs.SHORT_SHA }} --platform-filter=${{ inputs.PLATFORM_FILTER }}
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -68,7 +68,12 @@
     "python.analysis.extraPaths": [
         "rerun_py/rerun_sdk"
     ],
+    "C_Cpp.default.configurationProvider": "ms-vscode.cmake-tools", // Use cmake-tools to grab configs.
     "cmake.buildDirectory": "${workspaceRoot}/build/",
+    "cmake.buildTask": true, // Use a terminal instead of the output panel for builds since builds don't show colors in the output panel.
+    "cmake.environment": {
+        "CLICOLOR_FORCE": "1" // Force color output from cmake.
+    },
     "C_Cpp.autoAddFileAssociations": false,
     "rust-analyzer.showUnlinkedFileNotification": false,
     "black-formatter.args": [

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -55,6 +55,16 @@ if(NOT DEFINED CMAKE_CXX_STANDARD)
     set(CMAKE_CXX_STANDARD 17)
 endif()
 
+# Signal to all our build scripts that we're inside the Rerun repository.
+set(RERUN_REPOSITORY YES)
+
+# Set version number.
+# Read it from the Rust toml with a regex so we don't have another place to maintain!
+file(READ "Cargo.toml" CARGO_TOML)
+string(REGEX MATCH "\nversion = \"([a-z0-9\.\+-]+)\"\n" _ ${CARGO_TOML})
+set(RERUN_VERSION ${CMAKE_MATCH_1})
+message(INFO "Rerun version ${RERUN_VERSION}")
+
 # ------------------------------------------------------------------------------
 # Loguru logging library (https://github.com/emilk/loguru):
 set(CMAKE_DL_LIBS "dl") # Required by Loguru for backtraces

diff --git a/crates/rerun_c/src/rerun.h b/crates/rerun_c/src/rerun.h
@@ -1,4 +1,9 @@
-// The Rerun C SDK.
+// ----------------------------------------------------------------------------
+// The Rerun C SDK for Rerun version @RERUN_VERSION@
+// ----------------------------------------------------------------------------
+// This file is part of the rerun_c Rust crate.
+// EDITS TO COPIES OUTSIDE OF RERUN_C WILL BE OVERWRITTEN.
+// ----------------------------------------------------------------------------
 
 #ifndef RERUN_H
 #define RERUN_H
@@ -24,12 +29,12 @@ enum {
 /// Special value for `rr_recording_stream` methods to indicate the most appropriate
 /// globally available recording stream for recordings.
 /// (i.e. thread-local first, then global scope)
-#define RERUN_REC_STREAM_CURRENT_RECORDING ((rr_recording_stream)0xFFFFFFFF)
+#define RERUN_REC_STREAM_CURRENT_RECORDING 0xFFFFFFFF
 
 /// Special value for `rr_recording_stream` methods to indicate the most appropriate
 /// globally available recording stream for blueprints.
 /// (i.e. thread-local first, then global scope)
-#define RERUN_REC_STREAM_CURRENT_BLUEPRINT ((rr_recording_stream)0xFFFFFFFE)
+#define RERUN_REC_STREAM_CURRENT_BLUEPRINT 0xFFFFFFFE
 
 /// A unique handle for a recording stream.
 /// A recording stream handles everything related to logging data into Rerun.

diff --git a/examples/cpp/minimal/CMakeLists.txt b/examples/cpp/minimal/CMakeLists.txt
@@ -1,6 +1,27 @@
 cmake_minimum_required(VERSION 3.16)
 
-add_executable(example_minimal main.cpp)
+# If you use the example outside of the Rerun SDK you need to specify
+# where the rerun_c build is to be found by setting the `RERUN_CPP_URL` variable.
+# This can be done by passing `-DRERUN_CPP_URL=<path to rerun_sdk_cpp zip>` to cmake.
+if(DEFINED RERUN_REPOSITORY)
+    add_executable(example_minimal main.cpp)
+    set_default_warning_settings(example_minimal)
+else()
+    project(example_minimal LANGUAGES CXX)
 
-set_default_warning_settings(example_minimal)
+    add_executable(example_minimal main.cpp)
+
+    # Set the path to the rerun_c build.
+    set(RERUN_CPP_URL CACHE STRING "URL to the rerun_cpp zip.")
+
+    # Download the rerun_sdk
+    include(FetchContent)
+    FetchContent_Declare(rerun_sdk DOWNLOAD_EXTRACT_TIMESTAMP YES URL ${RERUN_CPP_URL})
+    FetchContent_MakeAvailable(rerun_sdk)
+
+    # Rerun requires at least C++17, but you can use an evern newer version.
+    set_property(TARGET example_minimal PROPERTY CXX_STANDARD 17)
+endif()
+
+# Link against rerun_sdk.
 target_link_libraries(example_minimal PRIVATE rerun_sdk)