Skip to content

Commit 2768fb3

Browse files
algoriddlefacebook-github-bot
algoriddle
authored and
facebook-github-bot
committed
faiss-gpu-raft package (facebookresearch#2992)
Summary: Pull Request resolved: facebookresearch#2992 Reviewed By: mdouze Differential Revision: D48391366 Pulled By: algoriddle fbshipit-source-id: 94b7f62afc8a09a9feaea47bf60e5358d89fcde5
1 parent c09992b commit 2768fb3

18 files changed

+283
-52
lines changed

‎.circleci/config.yml

+66-21
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,6 @@ executors:
1717
machine:
1818
image: ubuntu-2004:current
1919
resource_class: arm.medium
20-
macosx-x86_64-cpu:
21-
environment:
22-
CONDA_ARCH: MacOSX-x86_64
23-
macos:
24-
xcode: 11.7.0 # max supported for conda build, https://circleci.com/docs/using-macos#supported-xcode-versions
2520
macosx-arm64-cpu:
2621
environment:
2722
CONDA_ARCH: MacOSX-arm64
@@ -66,6 +61,9 @@ jobs:
6661
cuda:
6762
type: string
6863
default: ""
64+
raft:
65+
type: string
66+
default: ""
6967
cuda_archs:
7068
type: string
7169
default: ""
@@ -93,6 +91,8 @@ jobs:
9391
- run:
9492
name: Install conda build tools
9593
command: |
94+
conda config --set solver libmamba
95+
# conda config --set verbosity 3
9696
conda update -y -q conda
9797
conda install -y -q conda-build
9898
- when:
@@ -105,14 +105,16 @@ jobs:
105105
conda config --set anaconda_upload yes
106106
- when:
107107
condition:
108-
not: << parameters.label >>
108+
and:
109+
- not: << parameters.label >>
110+
- not: << parameters.cuda >>
109111
steps:
110112
- run:
111113
name: Conda build (CPU)
112114
no_output_timeout: 30m
113115
command: |
114116
cd conda
115-
conda build faiss --python 3.10 -c pytorch -c pkgs/main -c conda-forge
117+
conda build faiss --python 3.11 -c pytorch
116118
- when:
117119
condition:
118120
and:
@@ -124,12 +126,28 @@ jobs:
124126
no_output_timeout: 30m
125127
command: |
126128
cd conda
127-
conda build faiss --user pytorch --label <<parameters.label>> -c pytorch -c pkgs/main -c conda-forge
129+
conda build faiss --user pytorch --label <<parameters.label>> -c pytorch
130+
- when:
131+
condition:
132+
and:
133+
- not: << parameters.label >>
134+
- << parameters.cuda >>
135+
- not: << parameters.raft >>
136+
steps:
137+
- run:
138+
name: Conda build (GPU)
139+
no_output_timeout: 60m
140+
command: |
141+
sudo update-alternatives --set cuda /usr/local/cuda-<<parameters.cuda>>
142+
cd conda
143+
conda build faiss-gpu --variants '{ "cudatoolkit": "<<parameters.cuda>>", "c_compiler_version": "<<parameters.compiler_version>>", "cxx_compiler_version": "<<parameters.compiler_version>>" }' \
144+
-c pytorch -c nvidia
128145
- when:
129146
condition:
130147
and:
131148
- << parameters.label >>
132149
- << parameters.cuda >>
150+
- not: << parameters.raft >>
133151
steps:
134152
- run:
135153
name: Conda build (GPU) w/ anaconda upload
@@ -138,7 +156,37 @@ jobs:
138156
sudo update-alternatives --set cuda /usr/local/cuda-<<parameters.cuda>>
139157
cd conda
140158
conda build faiss-gpu --variants '{ "cudatoolkit": "<<parameters.cuda>>", "c_compiler_version": "<<parameters.compiler_version>>", "cxx_compiler_version": "<<parameters.compiler_version>>" }' \
141-
--user pytorch --label <<parameters.label>> -c pytorch -c nvidia -c pkgs/main -c conda-forge
159+
--user pytorch --label <<parameters.label>> -c pytorch -c nvidia
160+
- when:
161+
condition:
162+
and:
163+
- not: << parameters.label >>
164+
- << parameters.cuda >>
165+
- << parameters.raft >>
166+
steps:
167+
- run:
168+
name: Conda build (GPU w/ RAFT)
169+
no_output_timeout: 60m
170+
command: |
171+
sudo update-alternatives --set cuda /usr/local/cuda-<<parameters.cuda>>
172+
cd conda
173+
conda build faiss-gpu-raft --variants '{ "cudatoolkit": "<<parameters.cuda>>", "c_compiler_version": "<<parameters.compiler_version>>", "cxx_compiler_version": "<<parameters.compiler_version>>" }' \
174+
-c pytorch -c nvidia -c rapidsai -c conda-forge
175+
- when:
176+
condition:
177+
and:
178+
- << parameters.label >>
179+
- << parameters.cuda >>
180+
- << parameters.raft >>
181+
steps:
182+
- run:
183+
name: Conda build (GPU w/ RAFT) w/ anaconda upload
184+
no_output_timeout: 60m
185+
command: |
186+
sudo update-alternatives --set cuda /usr/local/cuda-<<parameters.cuda>>
187+
cd conda
188+
conda build faiss-gpu-raft --variants '{ "cudatoolkit": "<<parameters.cuda>>", "c_compiler_version": "<<parameters.compiler_version>>", "cxx_compiler_version": "<<parameters.compiler_version>>" }' \
189+
--user pytorch --label <<parameters.label>> -c pytorch -c nvidia -c rapidsai -c conda-forge
142190
143191
build_cmake:
144192
parameters:
@@ -180,7 +228,7 @@ jobs:
180228
command: |
181229
conda config --set solver libmamba
182230
conda update -y -q conda
183-
conda install -y -q pkgs/main::python=3.10 cmake make swig mkl=2023 mkl-devel=2023 numpy scipy pytest gxx_linux-64 sysroot_linux-64 -c pkgs/main -c conda-forge
231+
conda install -y -q python=3.11 cmake make swig mkl=2023 mkl-devel=2023 numpy scipy pytest gxx_linux-64 sysroot_linux-64
184232
- when:
185233
condition:
186234
equal: [ "ON", << parameters.raft >> ]
@@ -282,9 +330,6 @@ workflows:
282330
- build_conda:
283331
name: Linux x86_64 (conda)
284332
exec: linux-x86_64-cpu
285-
- build_conda:
286-
name: OSX x86_64 (conda)
287-
exec: macosx-x86_64-cpu
288333
- build_conda:
289334
name: Windows x86_64 (conda)
290335
exec: windows-x86_64-cpu
@@ -313,17 +358,21 @@ workflows:
313358
branches:
314359
ignore: /.*/
315360
- build_conda:
316-
name: Windows x86_64 packages
317-
exec: windows-x86_64-cpu
361+
name: Linux x86_64 GPU w/ RAFT packages (CUDA 11.4)
362+
exec: linux-x86_64-gpu
318363
label: main
364+
raft: "ON"
365+
cuda: "11.4"
366+
cuda_archs: "60;61;70;72;75;80;86"
367+
compiler_version: "11.2"
319368
filters:
320369
tags:
321370
only: /^v.*/
322371
branches:
323372
ignore: /.*/
324373
- build_conda:
325-
name: OSX x86_64 packages
326-
exec: macosx-x86_64-cpu
374+
name: Windows x86_64 packages
375+
exec: windows-x86_64-cpu
327376
label: main
328377
filters:
329378
tags:
@@ -373,10 +422,6 @@ workflows:
373422
name: Windows x86_64 nightlies
374423
exec: windows-x86_64-cpu
375424
label: nightly
376-
- build_conda:
377-
name: OSX x86_64 nightlies
378-
exec: macosx-x86_64-cpu
379-
label: nightly
380425
- build_conda:
381426
name: OSX arm64 nightlies
382427
exec: macosx-arm64-cpu

‎conda/conda_build_config.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
python:
2-
- 3.8 # [not x86_64 or not osx]
32
- 3.9
43
- 3.10
4+
- 3.11

‎conda/faiss-gpu-raft/build-lib.sh

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/sh
2+
# Copyright (c) Facebook, Inc. and its affiliates.
3+
#
4+
# This source code is licensed under the MIT license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
set -e
8+
9+
10+
# Build libfaiss.so/libfaiss_avx2.so.
11+
cmake -B _build \
12+
-DBUILD_SHARED_LIBS=ON \
13+
-DBUILD_TESTING=OFF \
14+
-DFAISS_OPT_LEVEL=avx2 \
15+
-DFAISS_ENABLE_GPU=ON \
16+
-DFAISS_ENABLE_RAFT=ON \
17+
-DCMAKE_CUDA_ARCHITECTURES="${CUDA_ARCHS}" \
18+
-DFAISS_ENABLE_PYTHON=OFF \
19+
-DBLA_VENDOR=Intel10_64lp \
20+
-DCMAKE_INSTALL_LIBDIR=lib \
21+
-DCMAKE_BUILD_TYPE=Release .
22+
23+
make -C _build -j$(nproc) faiss faiss_avx2
24+
25+
cmake --install _build --prefix $PREFIX
26+
cmake --install _build --prefix _libfaiss_stage/

‎conda/faiss-gpu-raft/build-pkg.sh

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/sh
2+
# Copyright (c) Facebook, Inc. and its affiliates.
3+
#
4+
# This source code is licensed under the MIT license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
set -e
8+
9+
10+
# Build swigfaiss.so/swigfaiss_avx2.so.
11+
cmake -B _build_python_${PY_VER} \
12+
-Dfaiss_ROOT=_libfaiss_stage/ \
13+
-DFAISS_OPT_LEVEL=avx2 \
14+
-DFAISS_ENABLE_GPU=ON \
15+
-DFAISS_ENABLE_RAFT=ON \
16+
-DCMAKE_BUILD_TYPE=Release \
17+
-DPython_EXECUTABLE=$PYTHON \
18+
faiss/python
19+
20+
make -C _build_python_${PY_VER} -j$(nproc) swigfaiss swigfaiss_avx2
21+
22+
# Build actual python module.
23+
cd _build_python_${PY_VER}/
24+
$PYTHON setup.py install --single-version-externally-managed --record=record.txt --prefix=$PREFIX

‎conda/faiss-gpu-raft/meta.yaml

+104
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
# Copyright (c) Facebook, Inc. and its affiliates.
2+
#
3+
# This source code is licensed under the MIT license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
{% set version = environ.get('GIT_DESCRIBE_TAG').lstrip('v') %}
7+
{% set suffix = "_nightly" if environ.get('PACKAGE_TYPE') == 'nightly' else "" %}
8+
{% set number = GIT_DESCRIBE_NUMBER %}
9+
10+
package:
11+
name: faiss-pkg
12+
version: {{ version }}
13+
14+
build:
15+
number: {{ number }}
16+
17+
about:
18+
home: https://github.com/facebookresearch/faiss
19+
license: MIT
20+
license_family: MIT
21+
license_file: LICENSE
22+
summary: A library for efficient similarity search and clustering of dense vectors.
23+
24+
source:
25+
git_url: ../../
26+
27+
outputs:
28+
- name: libfaiss
29+
script: build-lib.sh # [x86_64 and not win and not osx]
30+
script: build-lib-osx.sh # [x86_64 and osx]
31+
script: build-lib-arm64.sh # [not x86_64]
32+
script: build-lib.bat # [win]
33+
build:
34+
string: "h{{ PKG_HASH }}_{{ number }}_cuda{{ cudatoolkit }}_raft{{ suffix }}"
35+
run_exports:
36+
- {{ pin_compatible('libfaiss', exact=True) }}
37+
script_env:
38+
- CUDA_ARCHS
39+
requirements:
40+
build:
41+
- {{ compiler('cxx') }}
42+
- sysroot_linux-64 # [linux64]
43+
- llvm-openmp # [osx]
44+
- cmake >=3.23.1
45+
- make # [not win]
46+
- mkl-devel =2023 # [x86_64]
47+
host:
48+
- mkl =2023 # [x86_64]
49+
- openblas # [not x86_64]
50+
- cudatoolkit {{ cudatoolkit }}
51+
- libraft =23.08
52+
run:
53+
- mkl =2023 # [x86_64]
54+
- openblas # [not x86_64]
55+
- {{ pin_compatible('cudatoolkit', max_pin='x.x') }}
56+
- libraft =23.08
57+
test:
58+
requires:
59+
- conda-build
60+
commands:
61+
- test -f $PREFIX/lib/libfaiss$SHLIB_EXT # [not win]
62+
- test -f $PREFIX/lib/libfaiss_avx2$SHLIB_EXT # [x86_64 and not win]
63+
- conda inspect linkages -p $PREFIX $PKG_NAME # [not win]
64+
- conda inspect objects -p $PREFIX $PKG_NAME # [osx]
65+
66+
- name: faiss-gpu-raft
67+
script: build-pkg.sh # [x86_64 and not win and not osx]
68+
script: build-pkg-osx.sh # [x86_64 and osx]
69+
script: build-pkg-arm64.sh # [not x86_64]
70+
script: build-pkg.bat # [win]
71+
build:
72+
string: "py{{ PY_VER }}_h{{ PKG_HASH }}_{{ number }}_cuda{{ cudatoolkit }}{{ suffix }}"
73+
requirements:
74+
build:
75+
- {{ compiler('cxx') }}
76+
- sysroot_linux-64 =2.17 # [linux64]
77+
- swig
78+
- cmake >=3.23.1
79+
- make # [not win]
80+
host:
81+
- python {{ python }}
82+
- numpy >=1.19,<2
83+
- {{ pin_subpackage('libfaiss', exact=True) }}
84+
run:
85+
- python {{ python }}
86+
- numpy >=1.19,<2
87+
- {{ pin_subpackage('libfaiss', exact=True) }}
88+
test:
89+
requires:
90+
- numpy
91+
- scipy
92+
- pytorch
93+
commands:
94+
- python -X faulthandler -m unittest discover -v -s tests/ -p "test_*"
95+
- python -X faulthandler -m unittest discover -v -s tests/ -p "torch_*"
96+
- cp tests/common_faiss_tests.py faiss/gpu/test
97+
- python -X faulthandler -m unittest discover -v -s faiss/gpu/test/ -p "test_*"
98+
- python -X faulthandler -m unittest discover -v -s faiss/gpu/test/ -p "torch_*"
99+
- sh test_cpu_dispatch.sh # [linux64]
100+
files:
101+
- test_cpu_dispatch.sh # [linux64]
102+
source_files:
103+
- tests/
104+
- faiss/gpu/test/

‎conda/faiss-gpu/build-lib.sh

+1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ cmake -B _build \
1313
-DBUILD_TESTING=OFF \
1414
-DFAISS_OPT_LEVEL=avx2 \
1515
-DFAISS_ENABLE_GPU=ON \
16+
-DFAISS_ENABLE_RAFT=OFF \
1617
-DCMAKE_CUDA_ARCHITECTURES="${CUDA_ARCHS}" \
1718
-DFAISS_ENABLE_PYTHON=OFF \
1819
-DBLA_VENDOR=Intel10_64lp \

‎conda/faiss-gpu/build-pkg.sh

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ cmake -B _build_python_${PY_VER} \
1212
-Dfaiss_ROOT=_libfaiss_stage/ \
1313
-DFAISS_OPT_LEVEL=avx2 \
1414
-DFAISS_ENABLE_GPU=ON \
15+
-DFAISS_ENABLE_RAFT=OFF \
1516
-DCMAKE_BUILD_TYPE=Release \
1617
-DPython_EXECUTABLE=$PYTHON \
1718
faiss/python

‎conda/faiss-gpu/install-cmake.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,5 @@
66

77
set -e
88

9-
wget -O - https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.tar.gz | tar xzf -
10-
cp -R cmake-3.17.1-Linux-x86_64/* $PREFIX
9+
FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so
10+
LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so

‎conda/faiss-gpu/meta.yaml

+5-3
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ source:
2626

2727
outputs:
2828
- name: libfaiss
29-
script: build-lib.sh # [x86_64 and not win]
29+
script: build-lib.sh # [x86_64 and not win and not osx]
30+
script: build-lib-osx.sh # [x86_64 and osx]
3031
script: build-lib-arm64.sh # [not x86_64]
3132
script: build-lib.bat # [win]
3233
build:
@@ -38,7 +39,7 @@ outputs:
3839
requirements:
3940
build:
4041
- {{ compiler('cxx') }}
41-
- sysroot_linux-64 =2.17 # [linux64]
42+
- sysroot_linux-64 # [linux64]
4243
- llvm-openmp # [osx]
4344
- cmake >=3.23.1
4445
- make # [not win]
@@ -61,7 +62,8 @@ outputs:
6162
- conda inspect objects -p $PREFIX $PKG_NAME # [osx]
6263

6364
- name: faiss-gpu
64-
script: build-pkg.sh # [x86_64 and not win]
65+
script: build-pkg.sh # [x86_64 and not win and not osx]
66+
script: build-pkg-osx.sh # [x86_64 and osx]
6567
script: build-pkg-arm64.sh # [not x86_64]
6668
script: build-pkg.bat # [win]
6769
build:

0 commit comments

Comments
 (0)