Enable ROCm in build-only mode (#3713)

Summary:
Pull Request resolved: #3713

Enable ROCm in CI

Reviewed By: asadoughi

Differential Revision: D60598458

fbshipit-source-id: f95d9a2b0b894c3f7272d4f1963d9230e1069bf8

Author: ramilbakhshyiev

.github/actions/build_cmake/action.yml

@@ -12,6 +12,10 @@ inputs:
     description: 'Enable RAFT support.'
     required: false
     default: OFF
+  rocm:
+    description: 'Enable ROCm support.'
+    required: false
+    default: OFF
 runs:
   using: composite
   steps:
@@ -38,7 +42,11 @@ runs:
         # install base packages for X86_64
         if [ "${{ runner.arch }}" = "X64" ]; then
           # TODO: unpin versions for gxx_linux-64 and sysroot_linux-64 and merge it with ARM64 below
-          conda install -y -q -c conda-forge gxx_linux-64=11.2 sysroot_linux-64=2.28
+          if [ "${{ inputs.rocm }}" = "ON" ]; then
+            conda install -y -q -c conda-forge gxx_linux-64 sysroot_linux-64
+          else
+            conda install -y -q -c conda-forge gxx_linux-64=11.2 sysroot_linux-64=2.28
+          fi
           conda install -y -q mkl=2023 mkl-devel=2023
         fi
 
@@ -60,6 +68,48 @@ runs:
         else
           conda install -y -q pytorch -c pytorch
         fi
+    - name: ROCm - Install dependencies
+      if: inputs.rocm == 'ON'
+      shell: bash
+      run: |
+        # Update repos and install kmod, wget
+        sudo apt-get update
+        sudo apt-get install -y kmod wget
+
+        # Get UBUNTU version name
+        UBUNTU_VERSION_NAME=`cat /etc/os-release | grep UBUNTU_CODENAME | awk -F= '{print $2}'`
+
+        # Set ROCm version
+        ROCM_VERSION="6.1.1"
+
+        # Download, prepare, and install the package signing key
+        mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+
+        # Add rocm repository
+        wget -qO - http://repo.radeon.com/rocm/rocm.gpg.key | sudo apt-key add -
+        rocm_baseurl="http://repo.radeon.com/rocm/apt/${ROCM_VERSION}"
+        echo "deb [arch=amd64] ${rocm_baseurl} ${UBUNTU_VERSION_NAME} main" | sudo tee /etc/apt/sources.list.d/rocm.list
+        sudo apt-get update --allow-insecure-repositories
+        sudo apt-get install -y --allow-unauthenticated \
+            "rocm-dev${ROCM_VERSION}" "rocm-utils${ROCM_VERSION}" "rocm-libs${ROCM_VERSION}"
+
+        # Fake presence of MI200-class accelerators
+        echo "gfx90a" | sudo tee /opt/rocm/bin/target.lst
+
+        # Cleanup
+        sudo apt-get autoclean && sudo apt-get clean
+        sudo rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+        # symblink system libraries for HIP compiler
+        sudo ln -s /lib/x86_64-linux-gnu/libc.so.6 /lib64/libc.so.6
+        sudo ln -s /lib/x86_64-linux-gnu/libc_nonshared.a /usr/lib64/libc_nonshared.a
+        sudo ln -s /usr/lib/x86_64-linux-gnu/libpthread.so.0 /lib64/libpthread.so.0
+        sudo ln -s /home/runner/miniconda3/x86_64-conda-linux-gnu/sysroot/usr/lib64/libpthread_nonshared.a /usr/lib64/libpthread_nonshared.a
+    - name: ROCm - Hipify
+      if: inputs.rocm == 'ON'
+      shell: bash
+      run: ./faiss/gpu/hipify.sh
     - name: Build all targets
       shell: bash
       run: |
@@ -70,6 +120,7 @@ runs:
               -DBUILD_SHARED_LIBS=ON \
               -DFAISS_ENABLE_GPU=${{ inputs.gpu }} \
               -DFAISS_ENABLE_RAFT=${{ inputs.raft }} \
+              -DFAISS_ENABLE_ROCM=${{ inputs.rocm }} \
               -DFAISS_OPT_LEVEL=${{ inputs.opt_level }} \
               -DFAISS_ENABLE_C_API=ON \
               -DPYTHON_EXECUTABLE=$CONDA/bin/python \
@@ -79,6 +130,7 @@ runs:
               .
         make -k -C build -j$(nproc)
     - name: C++ tests
+      if: inputs.rocm == 'OFF'
       shell: bash
       run: |
         export GTEST_OUTPUT="xml:$(realpath .)/test-results/googletest/"
@@ -95,7 +147,7 @@ runs:
         pytest --junitxml=test-results/pytest/results.xml tests/test_*.py
         pytest --junitxml=test-results/pytest/results-torch.xml tests/torch_*.py
     - name: Python tests (CPU + GPU)
-      if: inputs.gpu == 'ON'
+      if: inputs.gpu == 'ON' && inputs.rocm == 'OFF'
       shell: bash
       run: |
         pytest --junitxml=test-results/pytest/results.xml tests/test_*.py
@@ -110,6 +162,7 @@ runs:
         FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs $CONDA/bin/python -c "import faiss" 2>&1 | grep faiss.so
         LD_DEBUG=libs $CONDA/bin/python -c "import faiss" 2>&1 | grep faiss_avx2.so
     - name: Upload test results
+      if: inputs.rocm == 'OFF'
       uses: actions/upload-artifact@v4
       with:
         name: test-results-${{ runner.arch }}-${{ inputs.opt_level }}-${{ inputs.gpu }}-${{ inputs.raft }}

.github/workflows/build.yml

@@ -88,6 +88,17 @@ jobs:
         with:
           gpu: ON
           raft: ON
+  linux-x86_64-GPU-w-ROCm-cmake:
+    name: Linux x86_64 GPU w/ ROCm (cmake)
+    needs: linux-x86_64-cmake
+    runs-on: 4-core-ubuntu
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: ./.github/actions/build_cmake
+        with:
+          gpu: ON
+          rocm: ON
   linux-arm64-SVE-cmake:
     name: Linux arm64 SVE (cmake)
     needs: linux-x86_64-cmake

CMakeLists.txt

@@ -23,10 +23,11 @@ cmake_minimum_required(VERSION 3.24.0 FATAL_ERROR)
 set(FAISS_LANGUAGES CXX)
 
 if(FAISS_ENABLE_GPU)
-  # if ROCm install detected, assume ROCm/HIP is GPU device
-  if (EXISTS /opt/rocm)
+  if (FAISS_ENABLE_ROCM)
     set(USE_ROCM TRUE)
     list(APPEND FAISS_LANGUAGES HIP)
+    list(PREPEND CMAKE_MODULE_PATH "/opt/rocm/lib/cmake")
+    list(PREPEND CMAKE_PREFIX_PATH "/opt/rocm")
   else()
     list(APPEND FAISS_LANGUAGES CUDA)
   endif()
@@ -60,6 +61,7 @@ list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
 option(FAISS_OPT_LEVEL "" "generic")
 option(FAISS_ENABLE_GPU "Enable support for GPU indexes." ON)
 option(FAISS_ENABLE_RAFT "Enable RAFT for GPU indexes." OFF)
+option(FAISS_ENABLE_ROCM "Enable ROCm for GPU indexes." OFF)
 option(FAISS_ENABLE_PYTHON "Build Python extension." ON)
 option(FAISS_ENABLE_C_API "Build C API." OFF)
 
@@ -78,8 +80,13 @@ if(FAISS_ENABLE_GPU)
 endif()
 
 if(FAISS_ENABLE_RAFT AND NOT TARGET raft::raft)
-   find_package(raft COMPONENTS compiled distributed)
- endif()
+  find_package(raft COMPONENTS compiled distributed)
+endif()
+
+if(USE_ROCM)
+  find_package(HIP REQUIRED)
+  find_package(hipBLAS REQUIRED)
+endif()
 
 add_subdirectory(faiss)
 

tests/CMakeLists.txt

@@ -123,6 +123,7 @@ target_link_libraries(faiss_test PRIVATE
   OpenMP::OpenMP_CXX
   GTest::gtest_main
   $<$<BOOL:${FAISS_ENABLE_RAFT}>:raft::raft>
+  $<$<BOOL:${FAISS_ENABLE_ROCM}>:hip::host>
 )
 
 # Defines `gtest_discover_tests()`.