Skip to content

Commit 0013c70

Browse files
algoriddlefacebook-github-bot
authored andcommitted
avx512 CI + conda packages (#3197)
Summary: Pull Request resolved: #3197 Reviewed By: mlomeli1 Differential Revision: D52689379 Pulled By: algoriddle fbshipit-source-id: 54e27c6d310d6da14777ae10ae62f46e7076cacf
1 parent 0710cbd commit 0013c70

17 files changed

+43
-35
lines changed

.circleci/config.yml

+4
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,10 @@ workflows:
314314
name: Linux x86_64 AVX2 (cmake)
315315
exec: linux-x86_64-cpu
316316
opt_level: "avx2"
317+
- build_cmake:
318+
name: Linux x86_64 AVX512 (cmake)
319+
exec: linux-x86_64-cpu
320+
opt_level: "avx512"
317321
- build_cmake:
318322
name: Linux x86_64 GPU (cmake)
319323
exec: linux-x86_64-gpu

.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,4 @@
1717
/tests/test
1818
/tests/gtest/
1919
faiss/python/swigfaiss_avx2.swig
20+
faiss/python/swigfaiss_avx512.swig

conda/faiss-gpu-raft/build-lib.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@
77
set -e
88

99

10-
# Build libfaiss.so/libfaiss_avx2.so.
10+
# Build libfaiss.so/libfaiss_avx2.so/libfaiss_avx512.so
1111
cmake -B _build \
1212
-DBUILD_SHARED_LIBS=ON \
1313
-DBUILD_TESTING=OFF \
14-
-DFAISS_OPT_LEVEL=avx2 \
14+
-DFAISS_OPT_LEVEL=avx512 \
1515
-DFAISS_ENABLE_GPU=ON \
1616
-DFAISS_ENABLE_RAFT=ON \
1717
-DCMAKE_CUDA_ARCHITECTURES="${CUDA_ARCHS}" \
@@ -20,7 +20,7 @@ cmake -B _build \
2020
-DCMAKE_INSTALL_LIBDIR=lib \
2121
-DCMAKE_BUILD_TYPE=Release .
2222

23-
make -C _build -j$(nproc) faiss faiss_avx2
23+
make -C _build -j$(nproc) faiss faiss_avx2 faiss_avx512
2424

2525
cmake --install _build --prefix $PREFIX
2626
cmake --install _build --prefix _libfaiss_stage/

conda/faiss-gpu-raft/build-pkg.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,17 @@
77
set -e
88

99

10-
# Build swigfaiss.so/swigfaiss_avx2.so.
10+
# Build swigfaiss.so/swigfaiss_avx2.so/swigfaiss_avx512.so
1111
cmake -B _build_python_${PY_VER} \
1212
-Dfaiss_ROOT=_libfaiss_stage/ \
13-
-DFAISS_OPT_LEVEL=avx2 \
13+
-DFAISS_OPT_LEVEL=avx512 \
1414
-DFAISS_ENABLE_GPU=ON \
1515
-DFAISS_ENABLE_RAFT=ON \
1616
-DCMAKE_BUILD_TYPE=Release \
1717
-DPython_EXECUTABLE=$PYTHON \
1818
faiss/python
1919

20-
make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2
20+
make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 swigfaiss_avx512
2121

2222
# Build actual python module.
2323
cd _build_python_${PY_VER}/

conda/faiss-gpu-raft/test_cpu_dispatch.sh

+3-2
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,6 @@
66

77
set -e
88

9-
FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so
10-
LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so
9+
FAISS_OPT_LEVEL= LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so
10+
FAISS_OPT_LEVEL=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so
11+
FAISS_OPT_LEVEL=AVX512 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx512.so

conda/faiss-gpu/build-lib.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@
77
set -e
88

99

10-
# Build libfaiss.so/libfaiss_avx2.so.
10+
# Build libfaiss.so/libfaiss_avx2.so/libfaiss_avx512.so
1111
cmake -B _build \
1212
-DBUILD_SHARED_LIBS=ON \
1313
-DBUILD_TESTING=OFF \
14-
-DFAISS_OPT_LEVEL=avx2 \
14+
-DFAISS_OPT_LEVEL=avx512 \
1515
-DFAISS_ENABLE_GPU=ON \
1616
-DFAISS_ENABLE_RAFT=OFF \
1717
-DCMAKE_CUDA_ARCHITECTURES="${CUDA_ARCHS}" \
@@ -20,7 +20,7 @@ cmake -B _build \
2020
-DCMAKE_INSTALL_LIBDIR=lib \
2121
-DCMAKE_BUILD_TYPE=Release .
2222

23-
make -C _build -j$(nproc) faiss faiss_avx2
23+
make -C _build -j$(nproc) faiss faiss_avx2 faiss_avx512
2424

2525
cmake --install _build --prefix $PREFIX
2626
cmake --install _build --prefix _libfaiss_stage/

conda/faiss-gpu/build-pkg.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,17 @@
77
set -e
88

99

10-
# Build swigfaiss.so/swigfaiss_avx2.so.
10+
# Build swigfaiss.so/swigfaiss_avx2.so/swigfaiss_avx512.so
1111
cmake -B _build_python_${PY_VER} \
1212
-Dfaiss_ROOT=_libfaiss_stage/ \
13-
-DFAISS_OPT_LEVEL=avx2 \
13+
-DFAISS_OPT_LEVEL=avx512 \
1414
-DFAISS_ENABLE_GPU=ON \
1515
-DFAISS_ENABLE_RAFT=OFF \
1616
-DCMAKE_BUILD_TYPE=Release \
1717
-DPython_EXECUTABLE=$PYTHON \
1818
faiss/python
1919

20-
make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2
20+
make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 swigfaiss_avx512
2121

2222
# Build actual python module.
2323
cd _build_python_${PY_VER}/

conda/faiss-gpu/test_cpu_dispatch.sh

+3-2
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,6 @@
66

77
set -e
88

9-
FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so
10-
LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so
9+
FAISS_OPT_LEVEL= LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so
10+
FAISS_OPT_LEVEL=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so
11+
FAISS_OPT_LEVEL=AVX512 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx512.so

conda/faiss/build-lib-osx.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@
77
set -e
88

99

10-
# Build libfaiss.so/libfaiss_avx2.so.
10+
# Build libfaiss.so/libfaiss_avx2.so/libfaiss_avx512.so
1111
cmake -B _build \
1212
-DBUILD_SHARED_LIBS=ON \
1313
-DBUILD_TESTING=OFF \
14-
-DFAISS_OPT_LEVEL=avx2 \
14+
-DFAISS_OPT_LEVEL=avx512 \
1515
-DFAISS_ENABLE_GPU=OFF \
1616
-DFAISS_ENABLE_PYTHON=OFF \
1717
-DBLA_VENDOR=Intel10_64lp \
@@ -21,7 +21,7 @@ cmake -B _build \
2121
-DCMAKE_INSTALL_LIBDIR=lib \
2222
-DCMAKE_BUILD_TYPE=Release .
2323

24-
make -C _build -j$(nproc) faiss faiss_avx2
24+
make -C _build -j$(nproc) faiss faiss_avx2 faiss_avx512
2525

2626
cmake --install _build --prefix $PREFIX
2727
cmake --install _build --prefix _libfaiss_stage/

conda/faiss/build-lib.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,18 @@
77
set -e
88

99

10-
# Build libfaiss.so/libfaiss_avx2.so.
10+
# Build libfaiss.so/libfaiss_avx2.so/libfaiss_avx512.so
1111
cmake -B _build \
1212
-DBUILD_SHARED_LIBS=ON \
1313
-DBUILD_TESTING=OFF \
14-
-DFAISS_OPT_LEVEL=avx2 \
14+
-DFAISS_OPT_LEVEL=avx512 \
1515
-DFAISS_ENABLE_GPU=OFF \
1616
-DFAISS_ENABLE_PYTHON=OFF \
1717
-DBLA_VENDOR=Intel10_64lp \
1818
-DCMAKE_INSTALL_LIBDIR=lib \
1919
-DCMAKE_BUILD_TYPE=Release .
2020

21-
make -C _build -j$(nproc) faiss faiss_avx2
21+
make -C _build -j$(nproc) faiss faiss_avx2 faiss_avx512
2222

2323
cmake --install _build --prefix $PREFIX
2424
cmake --install _build --prefix _libfaiss_stage/

conda/faiss/build-pkg-arm64.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
set -e
88

99

10-
# Build swigfaiss.so/swigfaiss_avx2.so.
10+
# Build swigfaiss.so
1111
cmake -B _build_python_${PY_VER} \
1212
-Dfaiss_ROOT=_libfaiss_stage/ \
1313
-DFAISS_ENABLE_GPU=OFF \

conda/faiss/build-pkg-osx.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@
77
set -e
88

99

10-
# Build swigfaiss.so/swigfaiss_avx2.so.
10+
# Build swigfaiss.so/swigfaiss_avx2.so/swigfaiss_avx512
1111
cmake -B _build_python_${PY_VER} \
1212
-Dfaiss_ROOT=_libfaiss_stage/ \
13-
-DFAISS_OPT_LEVEL=avx2 \
13+
-DFAISS_OPT_LEVEL=avx512 \
1414
-DFAISS_ENABLE_GPU=OFF \
1515
-DOpenMP_CXX_FLAGS=-fopenmp=libiomp5 \
1616
-DOpenMP_CXX_LIB_NAMES=libiomp5 \
@@ -19,7 +19,7 @@ cmake -B _build_python_${PY_VER} \
1919
-DPython_EXECUTABLE=$PYTHON \
2020
faiss/python
2121

22-
make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2
22+
make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 swigfaiss_avx512
2323

2424
# Build actual python module.
2525
cd _build_python_${PY_VER}/

conda/faiss/build-pkg.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,16 @@
77
set -e
88

99

10-
# Build swigfaiss.so/swigfaiss_avx2.so.
10+
# Build swigfaiss.so/swigfaiss_avx2.so/swigfaiss_avx512.so
1111
cmake -B _build_python_${PY_VER} \
1212
-Dfaiss_ROOT=_libfaiss_stage/ \
13-
-DFAISS_OPT_LEVEL=avx2 \
13+
-DFAISS_OPT_LEVEL=avx512 \
1414
-DFAISS_ENABLE_GPU=OFF \
1515
-DCMAKE_BUILD_TYPE=Release \
1616
-DPython_EXECUTABLE=$PYTHON \
1717
faiss/python
1818

19-
make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2
19+
make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2 swigfaiss_avx512
2020

2121
# Build actual python module.
2222
cd _build_python_${PY_VER}/

conda/faiss/test_cpu_dispatch.sh

+3-2
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,6 @@
66

77
set -e
88

9-
FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so
10-
LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so
9+
FAISS_OPT_LEVEL= LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so
10+
FAISS_OPT_LEVEL=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so
11+
FAISS_OPT_LEVEL=AVX512 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx512.so

faiss/CMakeLists.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,7 @@ if(FAISS_OPT_LEVEL STREQUAL "avx2")
328328
)
329329
endif()
330330
if(FAISS_OPT_LEVEL STREQUAL "avx512")
331-
install(TARGETS faiss_avx512
331+
install(TARGETS faiss_avx2 faiss_avx512
332332
EXPORT faiss-targets
333333
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
334334
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}

faiss/utils/utils.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ std::string get_compile_options() {
117117
#ifdef __AVX2__
118118
options += "AVX2 ";
119119
#elif __AVX512F__
120-
options += "AVX512";
120+
options += "AVX512 ";
121121
#elif defined(__aarch64__)
122122
options += "NEON ";
123123
#else

tests/test_index_accuracy.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -639,14 +639,14 @@ def test_OIVFPQ(self):
639639
d = ev.d
640640
quantizer = faiss.IndexFlatL2(d)
641641
index = faiss.IndexIVFPQ(quantizer, d, ncentroids, M, 8)
642-
index.nprobe = 5
642+
index.nprobe = 12
643643

644644
res = ev.launch("IVFPQ", index)
645645
e_ivfpq = ev.evalres(res)
646646

647647
quantizer = faiss.IndexFlatL2(d)
648648
index_ivfpq = faiss.IndexIVFPQ(quantizer, d, ncentroids, M, 8)
649-
index_ivfpq.nprobe = 5
649+
index_ivfpq.nprobe = 12
650650
opq_matrix = faiss.OPQMatrix(d, M)
651651
opq_matrix.niter = 10
652652
index = faiss.IndexPreTransform(opq_matrix, index_ivfpq)

0 commit comments

Comments
 (0)