.github/workflows/main.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.


# NOTE: must set "Crossbow" as name to have the badge links working in the
# github comment reports!
name: Crossbow
on: 
  workflow_dispatch:
    inputs:
      build_arm:
        description: 'Build ARM binary'
        required: true
        type: boolean
jobs:

  build-cpp-ubuntu:
    name: ${{ matrix.platform.title }}
    runs-on: ${{ matrix.platform.runs-on }}
    strategy:
      fail-fast: false
      matrix:
        platform:
          - runs-on: ubuntu-latest
            arch: amd64
            clang-tools: "14"
            llvm: "14"
            title: AMD64 Ubuntu 22.04
            ubuntu: "22.04"
            isARM: false
          - runs-on: ubuntu-latest 
            isARM: ${{ inputs.build_arm }}
            arch: aarch_64
            clang-tools: "14"
            llvm: "14"
            title: AMD64 Ubuntu 22.04
            ubuntu: "22.04"
        exclude: 
          - {platform: {isARM: true}}
          # - arch: arm64v8
          #   clang-tools: "10"
          #   llvm: "10"
          #   runs-on: buildjet-4vcpu-ubuntu-2204-arm
          #   title: ARM64 Ubuntu 20.04
          #   ubuntu: "20.04"
    env:
      ARCH: ${{ matrix.platform.arch }}
      CLANG_TOOLS: ${{ matrix.platform.clang-tools }}
      LLVM: ${{ matrix.platform.llvm }}
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v3
        with:
          fetch-depth: 1
          path: arrow
          ref: testBuild
          submodules: recursive

      - name: Free up disk space
        if: runner.os == 'Linux' && runner.arch == 'X64'
        shell: bash
        run: |
          df -h
          echo "::group::/usr/local/*"
          du -hsc /usr/local/*
          echo "::endgroup::"
          echo "::group::/usr/local/bin/*"
          du -hsc /usr/local/bin/*
          echo "::endgroup::"
          # ~1GB (From 1.2GB to 214MB)
          sudo rm -rf \
            /usr/local/bin/aliyun \
            /usr/local/bin/azcopy \
            /usr/local/bin/bicep \
            /usr/local/bin/cmake-gui \
            /usr/local/bin/cpack \
            /usr/local/bin/helm \
            /usr/local/bin/hub \
            /usr/local/bin/kubectl \
            /usr/local/bin/minikube \
            /usr/local/bin/node \
            /usr/local/bin/packer \
            /usr/local/bin/pulumi* \
            /usr/local/bin/stack \
            /usr/local/bin/terraform || :
          echo "::group::/usr/local/share/*"
          du -hsc /usr/local/share/*
          echo "::endgroup::"
          # 1.3GB
          sudo rm -rf /usr/local/share/powershell || :
          echo "::group::/opt/*"
          du -hsc /opt/*
          echo "::endgroup::"
          echo "::group::/opt/hostedtoolcache/*"
          du -hsc /opt/hostedtoolcache/*
          echo "::endgroup::"
          # 5.3GB
          sudo rm -rf /opt/hostedtoolcache/CodeQL || :
          # 1.4GB
          sudo rm -rf /opt/hostedtoolcache/go || :
          # 489MB
          sudo rm -rf /opt/hostedtoolcache/PyPy || :
          # 376MB
          sudo rm -rf /opt/hostedtoolcache/node || :
          # Remove Web browser packages
          sudo apt-get purge -y -f firefox \
                  google-chrome-stable \
                  microsoft-edge-stable
          df -h
      - name: Set up Python by actions/setup-python
        if: runner.arch == 'X64'
        uses: actions/setup-python@v4
        with:
          cache: 'pip'
          python-version: 3.8
      - name: Set up Python by apt
        if: runner.os == 'Linux' && runner.arch != 'X64'
        run: |
          sudo apt update
          sudo apt-get install -y python3-pip
          pip install -U pip
          echo "$HOME/.local/bin" >>"$GITHUB_PATH"
      - name: Install Archery
        shell: bash
        run: pip install -e arrow/dev/archery[all]

      - name: Free up disk space
        run: |
            df -h
            sudo rm -rf /opt/hostedtoolcache/CodeQL || :
            df -h
      - name: Set Swap Space
        uses: thejerrybao/setup-swap-space@v1
        with:
          swap-space-path: /swapfile
          swap-size-gb: 16
          remove-existing-swap-files: false
      - name: Build C++ libraries
        # env:
        #   AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
        #   AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
        #   SCCACHE_BUCKET: "${{ secrets.SCCACHE_BUCKET }}"
        #   SCCACHE_REGION: "${{ secrets.SCCACHE_REGION }}"
        #   SCCACHE_S3_KEY_PREFIX: "sccache"

        run: |
          archery docker run \
            -e ARROW_JAVA_BUILD=OFF \
            -e ARROW_JAVA_TEST=OFF \
            java-jni-manylinux-2014
      - name: Compress into single artifact to keep directory structure
        run: tar -cvzf arrow-shared-libs-linux.tar.gz arrow/java-dist/
      - name: Upload artifacts
        uses: actions/upload-artifact@v2
        with:
          name: ubuntu-shared-lib
          path: arrow-shared-libs-linux.tar.gz

  # build-cpp-macos:
  #   name: Build C++ libraries macOS ${{ matrix.platform.arch }}
  #   runs-on: ${{ matrix.platform.runs_on }}
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       platform:
  #         - { runs_on: ["macos-latest"], arch: "x86_64"}
  #   env:
  #     MACOSX_DEPLOYMENT_TARGET: "10.13"
  #   steps:
  #     - name: Checkout Arrow
  #       uses: actions/checkout@v3
  #       with:
  #         fetch-depth: 1
  #         path: arrow
  #         ref: testBuild
  #         submodules: recursive

  #     - name: Set up Python
  #       if: |
  #         !contains(matrix.platform.runs_on, 'self-hosted')
  #       uses: actions/setup-python@v4
  #       with:
  #         cache: 'pip'
  #         python-version: 3.8
  #     - name: Install Archery
  #       shell: bash
  #       run: pip install -e arrow/dev/archery[all]
  #     - name: Install dependencies
  #       run: |
  #         # We want to use llvm@14 to avoid shared z3
  #         # dependency. llvm@14 doesn't depend on z3 and llvm depends
  #         # on z3. And Homebrew's z3 provides only shared library. It
  #         # doesn't provides static z3 because z3's CMake doesn't accept
  #         # building both shared and static libraries at once.
  #         # See also: Z3_BUILD_LIBZ3_SHARED in
  #         # https://github.com/Z3Prover/z3/blob/master/README-CMake.md
  #         #
  #         # If llvm is installed, Apache Arrow C++ uses llvm rather than
  #         # llvm@14 because llvm is newer than llvm@14.
  #         brew uninstall llvm || :
  #         brew bundle --file=arrow/cpp/Brewfile
  #         # We want to link aws-sdk-cpp statically but Homebrew's
  #         # aws-sdk-cpp provides only shared library. If we have
  #         # Homebrew's aws-sdk-cpp, our build mix Homebrew's
  #         # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
  #         # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
  #         brew uninstall aws-sdk-cpp
  #         # We want to use bundled RE2 for static linking. If
  #         # Homebrew's RE2 is installed, its header file may be used.
  #         # We uninstall Homebrew's RE2 to ensure using bundled RE2.
  #         brew uninstall grpc || : # gRPC depends on RE2
  #         brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too
  #         brew uninstall re2
  #         # We want to use bundled Protobuf for static linking. If
  #         # Homebrew's Protobuf is installed, its library file may be
  #         # used on test  We uninstall Homebrew's Protobuf to ensure using
  #         # bundled Protobuf.
  #         brew uninstall protobuf
  #         brew bundle --file=arrow/java/Brewfile
  #     - name: Build C++ libraries
  #       env:
  #         ARROW_USE_CCACHE: "ON"
  #       run: |
  #         set -e
  #         # make brew Java available to CMake
  #         if [ "${{ matrix.platform.arch }}" = "aarch_64" ]; then
  #           export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
  #         fi
  #         arrow/ci/scripts/java_jni_macos_build.sh \
  #           $GITHUB_WORKSPACE/arrow \
  #           $GITHUB_WORKSPACE/arrow/cpp-build \
  #           $GITHUB_WORKSPACE/arrow/java-dist
  #     - name: Compress into single artifact to keep directory structure
  #       run: tar -cvzf arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz arrow/java-dist/
  #     - name: Upload artifacts
  #       uses: actions/upload-artifact@v2
  #       with:
  #         name: macos-shared-lib-${{ matrix.platform.arch }}
  #         path: arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz

  package-jars:
    name: Build jar files
    runs-on: macos-latest
    needs:
      - build-cpp-ubuntu
      # - build-cpp-macos
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
          path: arrow
          ref: testBuild
          submodules: recursive

      - name: Download Libraries
        uses: actions/download-artifact@v3
        with:
          path: artifacts
      - name: Decompress artifacts
        run: |
          mv artifacts/*/*.tar.gz .
          tar -xvzf arrow-shared-libs-linux.tar.gz
          tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz
      - name: Test that shared libraries exist
        run: |
          set -x
          test -f arrow/java-dist/x86_64/libarrow_cdata_jni.so
          test -f arrow/java-dist/x86_64/libarrow_dataset_jni.so
          test -f arrow/java-dist/x86_64/libarrow_orc_jni.so
          test -f arrow/java-dist/x86_64/libgandiva_jni.so
          test -f arrow/java-dist/x86_64/libarrow_cdata_jni.dylib
          test -f arrow/java-dist/x86_64/libarrow_dataset_jni.dylib
          test -f arrow/java-dist/x86_64/libarrow_orc_jni.dylib
          test -f arrow/java-dist/x86_64/libgandiva_jni.dylib
      - name: Build bundled jar
        run: |
          set -e
          pushd arrow/java
          mvn versions:set -DnewVersion=13.0.0-SNAPSHOT
          popd
          arrow/ci/scripts/java_full_build.sh \
            $GITHUB_WORKSPACE/arrow \
            $GITHUB_WORKSPACE/arrow/java-dist
      - name: Upload artifacts
        uses: actions/upload-artifact@v3
        with:
          name: arrow-jar
          path: arrow/java-dist/arrow-gandiva-13.0.0-SNAPSHOT.jar