avx512 CI + conda packages (#3197)

Summary: Pull Request resolved: #3197

Reviewed By: mlomeli1

Differential Revision: D52689379

Pulled By: algoriddle

fbshipit-source-id: 54e27c6d310d6da14777ae10ae62f46e7076cacf

Author: algoriddle

.circleci/config.yml

@@ -314,6 +314,10 @@ workflows:
           name: Linux x86_64 AVX2 (cmake)
           exec: linux-x86_64-cpu
           opt_level: "avx2"
+      - build_cmake:
+          name: Linux x86_64 AVX512 (cmake)
+          exec: linux-x86_64-cpu
+          opt_level: "avx512"
       - build_cmake:
           name: Linux x86_64 GPU (cmake)
           exec: linux-x86_64-gpu

.gitignore

@@ -17,3 +17,4 @@
 /tests/test
 /tests/gtest/
 faiss/python/swigfaiss_avx2.swig
+faiss/python/swigfaiss_avx512.swig

conda/faiss-gpu-raft/build-lib.sh

@@ -7,11 +7,11 @@
 set -e
 
 
-# Build libfaiss.so/libfaiss_avx2.so.
+# Build libfaiss.so/libfaiss_avx2.so/libfaiss_avx512.so
 cmake -B _build \
       -DBUILD_SHARED_LIBS=ON \
       -DBUILD_TESTING=OFF \
-      -DFAISS_OPT_LEVEL=avx2 \
+      -DFAISS_OPT_LEVEL=avx512 \
       -DFAISS_ENABLE_GPU=ON \
       -DFAISS_ENABLE_RAFT=ON \
       -DCMAKE_CUDA_ARCHITECTURES="${CUDA_ARCHS}" \
@@ -20,7 +20,7 @@ cmake -B _build \
       -DCMAKE_INSTALL_LIBDIR=lib \
       -DCMAKE_BUILD_TYPE=Release .
 
-make -C _build -j$(nproc) faiss faiss_avx2
+make -C _build -j$(nproc) faiss faiss_avx2 faiss_avx512
 
 cmake --install _build --prefix $PREFIX
 cmake --install _build --prefix _libfaiss_stage/

conda/faiss-gpu-raft/build-pkg.sh

@@ -7,17 +7,17 @@
 set -e
 
 
-# Build swigfaiss.so/swigfaiss_avx2.so.
+# Build swigfaiss.so/swigfaiss_avx2.so/swigfaiss_avx512.so
 cmake -B _build_python_${PY_VER} \
       -Dfaiss_ROOT=_libfaiss_stage/ \
-      -DFAISS_OPT_LEVEL=avx2 \
+      -DFAISS_OPT_LEVEL=avx512 \
       -DFAISS_ENABLE_GPU=ON \
       -DFAISS_ENABLE_RAFT=ON \
       -DCMAKE_BUILD_TYPE=Release \
       -DPython_EXECUTABLE=$PYTHON \
       faiss/python
 
-make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2
+make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 swigfaiss_avx512
 
 # Build actual python module.
 cd _build_python_${PY_VER}/

conda/faiss-gpu-raft/test_cpu_dispatch.sh

@@ -6,5 +6,6 @@
 
 set -e
 
-FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so
-LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so
+FAISS_OPT_LEVEL= LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so
+FAISS_OPT_LEVEL=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so
+FAISS_OPT_LEVEL=AVX512 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx512.so

conda/faiss-gpu/build-lib.sh

@@ -7,11 +7,11 @@
 set -e
 
 
-# Build libfaiss.so/libfaiss_avx2.so.
+# Build libfaiss.so/libfaiss_avx2.so/libfaiss_avx512.so
 cmake -B _build \
       -DBUILD_SHARED_LIBS=ON \
       -DBUILD_TESTING=OFF \
-      -DFAISS_OPT_LEVEL=avx2 \
+      -DFAISS_OPT_LEVEL=avx512 \
       -DFAISS_ENABLE_GPU=ON \
       -DFAISS_ENABLE_RAFT=OFF \
       -DCMAKE_CUDA_ARCHITECTURES="${CUDA_ARCHS}" \
@@ -20,7 +20,7 @@ cmake -B _build \
       -DCMAKE_INSTALL_LIBDIR=lib \
       -DCMAKE_BUILD_TYPE=Release .
 
-make -C _build -j$(nproc) faiss faiss_avx2
+make -C _build -j$(nproc) faiss faiss_avx2 faiss_avx512
 
 cmake --install _build --prefix $PREFIX
 cmake --install _build --prefix _libfaiss_stage/

conda/faiss-gpu/build-pkg.sh

@@ -7,17 +7,17 @@
 set -e
 
 
-# Build swigfaiss.so/swigfaiss_avx2.so.
+# Build swigfaiss.so/swigfaiss_avx2.so/swigfaiss_avx512.so
 cmake -B _build_python_${PY_VER} \
       -Dfaiss_ROOT=_libfaiss_stage/ \
-      -DFAISS_OPT_LEVEL=avx2 \
+      -DFAISS_OPT_LEVEL=avx512 \
       -DFAISS_ENABLE_GPU=ON \
       -DFAISS_ENABLE_RAFT=OFF \
       -DCMAKE_BUILD_TYPE=Release \
       -DPython_EXECUTABLE=$PYTHON \
       faiss/python
 
-make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2
+make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 swigfaiss_avx512
 
 # Build actual python module.
 cd _build_python_${PY_VER}/

conda/faiss-gpu/test_cpu_dispatch.sh

@@ -6,5 +6,6 @@
 
 set -e
 
-FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so
-LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so
+FAISS_OPT_LEVEL= LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so
+FAISS_OPT_LEVEL=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so
+FAISS_OPT_LEVEL=AVX512 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx512.so

conda/faiss/build-lib-osx.sh

@@ -7,11 +7,11 @@
 set -e
 
 
-# Build libfaiss.so/libfaiss_avx2.so.
+# Build libfaiss.so/libfaiss_avx2.so/libfaiss_avx512.so
 cmake -B _build \
       -DBUILD_SHARED_LIBS=ON \
       -DBUILD_TESTING=OFF \
-      -DFAISS_OPT_LEVEL=avx2 \
+      -DFAISS_OPT_LEVEL=avx512 \
       -DFAISS_ENABLE_GPU=OFF \
       -DFAISS_ENABLE_PYTHON=OFF \
       -DBLA_VENDOR=Intel10_64lp \
@@ -21,7 +21,7 @@ cmake -B _build \
       -DCMAKE_INSTALL_LIBDIR=lib \
       -DCMAKE_BUILD_TYPE=Release .
 
-make -C _build -j$(nproc) faiss faiss_avx2
+make -C _build -j$(nproc) faiss faiss_avx2 faiss_avx512
 
 cmake --install _build --prefix $PREFIX
 cmake --install _build --prefix _libfaiss_stage/

conda/faiss/build-lib.sh

@@ -7,18 +7,18 @@
 set -e
 
 
-# Build libfaiss.so/libfaiss_avx2.so.
+# Build libfaiss.so/libfaiss_avx2.so/libfaiss_avx512.so
 cmake -B _build \
       -DBUILD_SHARED_LIBS=ON \
       -DBUILD_TESTING=OFF \
-      -DFAISS_OPT_LEVEL=avx2 \
+      -DFAISS_OPT_LEVEL=avx512 \
       -DFAISS_ENABLE_GPU=OFF \
       -DFAISS_ENABLE_PYTHON=OFF \
       -DBLA_VENDOR=Intel10_64lp \
       -DCMAKE_INSTALL_LIBDIR=lib \
       -DCMAKE_BUILD_TYPE=Release .
 
-make -C _build -j$(nproc) faiss faiss_avx2
+make -C _build -j$(nproc) faiss faiss_avx2 faiss_avx512
 
 cmake --install _build --prefix $PREFIX
 cmake --install _build --prefix _libfaiss_stage/

conda/faiss/build-pkg-arm64.sh

@@ -7,7 +7,7 @@
 set -e
 
 
-# Build swigfaiss.so/swigfaiss_avx2.so.
+# Build swigfaiss.so
 cmake -B _build_python_${PY_VER} \
       -Dfaiss_ROOT=_libfaiss_stage/ \
       -DFAISS_ENABLE_GPU=OFF \

conda/faiss/build-pkg-osx.sh

@@ -7,10 +7,10 @@
 set -e
 
 
-# Build swigfaiss.so/swigfaiss_avx2.so.
+# Build swigfaiss.so/swigfaiss_avx2.so/swigfaiss_avx512
 cmake -B _build_python_${PY_VER} \
       -Dfaiss_ROOT=_libfaiss_stage/ \
-      -DFAISS_OPT_LEVEL=avx2 \
+      -DFAISS_OPT_LEVEL=avx512 \
       -DFAISS_ENABLE_GPU=OFF \
       -DOpenMP_CXX_FLAGS=-fopenmp=libiomp5 \
       -DOpenMP_CXX_LIB_NAMES=libiomp5 \
@@ -19,7 +19,7 @@ cmake -B _build_python_${PY_VER} \
       -DPython_EXECUTABLE=$PYTHON \
       faiss/python
 
-make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2
+make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 swigfaiss_avx512
 
 # Build actual python module.
 cd _build_python_${PY_VER}/

conda/faiss/build-pkg.sh

@@ -7,16 +7,16 @@
 set -e
 
 
-# Build swigfaiss.so/swigfaiss_avx2.so.
+# Build swigfaiss.so/swigfaiss_avx2.so/swigfaiss_avx512.so
 cmake -B _build_python_${PY_VER} \
       -Dfaiss_ROOT=_libfaiss_stage/ \
-      -DFAISS_OPT_LEVEL=avx2 \
+      -DFAISS_OPT_LEVEL=avx512 \
       -DFAISS_ENABLE_GPU=OFF \
       -DCMAKE_BUILD_TYPE=Release \
       -DPython_EXECUTABLE=$PYTHON \
       faiss/python
 
-make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2
+make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 swigfaiss_avx512
 
 # Build actual python module.
 cd _build_python_${PY_VER}/

conda/faiss/test_cpu_dispatch.sh

@@ -6,5 +6,6 @@
 
 set -e
 
-FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so
-LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so
+FAISS_OPT_LEVEL= LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so
+FAISS_OPT_LEVEL=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so
+FAISS_OPT_LEVEL=AVX512 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx512.so

faiss/CMakeLists.txt

@@ -328,7 +328,7 @@ if(FAISS_OPT_LEVEL STREQUAL "avx2")
   )
 endif()
 if(FAISS_OPT_LEVEL STREQUAL "avx512")
-  install(TARGETS faiss_avx512
+  install(TARGETS faiss_avx2 faiss_avx512
     EXPORT faiss-targets
     ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
     LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}

faiss/utils/utils.cpp

@@ -117,7 +117,7 @@ std::string get_compile_options() {
 #ifdef __AVX2__
     options += "AVX2 ";
 #elif __AVX512F__
-    options += "AVX512";
+    options += "AVX512 ";
 #elif defined(__aarch64__)
     options += "NEON ";
 #else

tests/test_index_accuracy.py

@@ -639,14 +639,14 @@ def test_OIVFPQ(self):
         d = ev.d
         quantizer = faiss.IndexFlatL2(d)
         index = faiss.IndexIVFPQ(quantizer, d, ncentroids, M, 8)
-        index.nprobe = 5
+        index.nprobe = 12
 
         res = ev.launch("IVFPQ", index)
         e_ivfpq = ev.evalres(res)
 
         quantizer = faiss.IndexFlatL2(d)
         index_ivfpq = faiss.IndexIVFPQ(quantizer, d, ncentroids, M, 8)
-        index_ivfpq.nprobe = 5
+        index_ivfpq.nprobe = 12
         opq_matrix = faiss.OPQMatrix(d, M)
         opq_matrix.niter = 10
         index = faiss.IndexPreTransform(opq_matrix, index_ivfpq)