Skip to content

Commit cb1a512

Browse files
Michael Norrisfacebook-github-bot
Michael Norris
authored andcommitted
Back out "Add example of how to build, link, and test an external SWIG module" (#3954)
Summary: Pull Request resolved: #3954 Original commit changeset: 0c1cd25eabbf Original Phabricator Diff: D63991471 Reviewed By: mengdilin, asadoughi Differential Revision: D64439886 fbshipit-source-id: cc90958f6d90a429a4eece8e1cd1322b20d9aceb
1 parent dce7c09 commit cb1a512

File tree

9 files changed

+29
-309
lines changed

9 files changed

+29
-309
lines changed

conda/faiss-gpu-raft/meta.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ outputs:
112112
- pytorch
113113
- pytorch-cuda {{ cuda_constraints }}
114114
commands:
115-
- python -X faulthandler -m unittest discover -v -s tests/ -p "(?!.*test_external_module\.py)test_.*py"
115+
- python -X faulthandler -m unittest discover -v -s tests/ -p "test_*"
116116
- python -X faulthandler -m unittest discover -v -s tests/ -p "torch_*"
117117
- cp tests/common_faiss_tests.py faiss/gpu/test
118118
- python -X faulthandler -m unittest discover -v -s faiss/gpu/test/ -p "test_*"

conda/faiss-gpu/meta.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ outputs:
101101
- pytorch
102102
- pytorch-cuda {{ cuda_constraints }}
103103
commands:
104-
- python -X faulthandler -m unittest discover -v -s tests/ -p "(?!.*test_external_module\.py)test_.*py"
104+
- python -X faulthandler -m unittest discover -v -s tests/ -p "test_*"
105105
- python -X faulthandler -m unittest discover -v -s tests/ -p "torch_*"
106106
- cp tests/common_faiss_tests.py faiss/gpu/test
107107
- python -X faulthandler -m unittest discover -v -s faiss/gpu/test/ -p "test_*"

conda/faiss/meta.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ outputs:
8686
- scipy
8787
- pytorch
8888
commands:
89-
- python -X faulthandler -m unittest discover -v -s tests/ -p "(?!.*test_external_module\.py)test_.*py"
89+
- python -X faulthandler -m unittest discover -v -s tests/ -p "test_*"
9090
- python -X faulthandler -m unittest discover -v -s tests/ -p "torch_*"
9191
- sh test_cpu_dispatch.sh # [linux64]
9292
files:

faiss/python/CMakeLists.txt

-29
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,6 @@ configure_swigfaiss(swigfaiss.swig)
6161
configure_swigfaiss(swigfaiss_avx2.swig)
6262
configure_swigfaiss(swigfaiss_avx512.swig)
6363
configure_swigfaiss(swigfaiss_sve.swig)
64-
configure_swigfaiss(faiss_example_external_module.swig)
6564

6665
if(TARGET faiss)
6766
# Manually add headers as extra dependencies of swigfaiss.
@@ -75,8 +74,6 @@ if(TARGET faiss)
7574
"${faiss_SOURCE_DIR}/faiss/${h}")
7675
list(APPEND SWIG_MODULE_swigfaiss_sve_EXTRA_DEPS
7776
"${faiss_SOURCE_DIR}/faiss/${h}")
78-
list(APPEND SWIG_MODULE_faiss_example_external_module_EXTRA_DEPS
79-
"${faiss_SOURCE_DIR}/faiss/${h}")
8077
endforeach()
8178
if(FAISS_ENABLE_ROCM)
8279
foreach(h ${FAISS_GPU_HEADERS})
@@ -86,8 +83,6 @@ if(TARGET faiss)
8683
"${faiss_SOURCE_DIR}/faiss/gpu-rocm/${h}")
8784
list(APPEND SWIG_MODULE_swigfaiss_avx512_EXTRA_DEPS
8885
"${faiss_SOURCE_DIR}/faiss/gpu-rocm/${h}")
89-
list(APPEND SWIG_MODULE_faiss_example_external_module_EXTRA_DEPS
90-
"${faiss_SOURCE_DIR}/faiss/gpu-rocm/${h}")
9186
endforeach()
9287
else()
9388
foreach(h ${FAISS_GPU_HEADERS})
@@ -99,8 +94,6 @@ if(TARGET faiss)
9994
"${faiss_SOURCE_DIR}/faiss/gpu/${h}")
10095
list(APPEND SWIG_MODULE_swigfaiss_sve_EXTRA_DEPS
10196
"${faiss_SOURCE_DIR}/faiss/gpu/${h}")
102-
list(APPEND SWIG_MODULE_faiss_example_external_module_EXTRA_DEPS
103-
"${faiss_SOURCE_DIR}/faiss/gpu/${h}")
10497
endforeach()
10598
endif()
10699
else()
@@ -159,37 +152,25 @@ if(NOT FAISS_OPT_LEVEL STREQUAL "sve")
159152
set_target_properties(swigfaiss_sve PROPERTIES EXCLUDE_FROM_ALL TRUE)
160153
endif()
161154

162-
set_property(SOURCE faiss_example_external_module.swig
163-
PROPERTY SWIG_MODULE_NAME faiss_example_external_module)
164-
swig_add_library(faiss_example_external_module
165-
TYPE SHARED
166-
LANGUAGE python
167-
SOURCES faiss_example_external_module.swig
168-
)
169-
set_property(TARGET faiss_example_external_module PROPERTY SWIG_COMPILE_OPTIONS -doxygen)
170-
171155
if(NOT WIN32)
172156
# NOTE: Python does not recognize the dylib extension.
173157
set_target_properties(swigfaiss PROPERTIES SUFFIX .so)
174158
set_target_properties(swigfaiss_avx2 PROPERTIES SUFFIX .so)
175159
set_target_properties(swigfaiss_avx512 PROPERTIES SUFFIX .so)
176160
set_target_properties(swigfaiss_sve PROPERTIES SUFFIX .so)
177-
set_target_properties(faiss_example_external_module PROPERTIES SUFFIX .so)
178161
else()
179162
# we need bigobj for the swig wrapper
180163
target_compile_options(swigfaiss PRIVATE /bigobj)
181164
target_compile_options(swigfaiss_avx2 PRIVATE /bigobj)
182165
target_compile_options(swigfaiss_avx512 PRIVATE /bigobj)
183166
target_compile_options(swigfaiss_sve PRIVATE /bigobj)
184-
target_compile_options(faiss_example_external_module PRIVATE /bigobj)
185167
endif()
186168

187169
if(FAISS_ENABLE_GPU)
188170
if(FAISS_ENABLE_ROCM)
189171
target_link_libraries(swigfaiss PRIVATE hip::host)
190172
target_link_libraries(swigfaiss_avx2 PRIVATE hip::host)
191173
target_link_libraries(swigfaiss_avx512 PRIVATE hip::host)
192-
target_link_libraries(faiss_example_external_module PRIVATE hip::host)
193174
else()
194175
find_package(CUDAToolkit REQUIRED)
195176
if(FAISS_ENABLE_RAFT)
@@ -240,21 +221,12 @@ target_link_libraries(swigfaiss_sve PRIVATE
240221
OpenMP::OpenMP_CXX
241222
)
242223

243-
target_link_libraries(faiss_example_external_module PRIVATE
244-
Python::Module
245-
Python::NumPy
246-
OpenMP::OpenMP_CXX
247-
swigfaiss
248-
faiss
249-
)
250-
251224
# Hack so that python_callbacks.h can be included as
252225
# `#include <faiss/python/python_callbacks.h>`.
253226
target_include_directories(swigfaiss PRIVATE ${PROJECT_SOURCE_DIR}/../..)
254227
target_include_directories(swigfaiss_avx2 PRIVATE ${PROJECT_SOURCE_DIR}/../..)
255228
target_include_directories(swigfaiss_avx512 PRIVATE ${PROJECT_SOURCE_DIR}/../..)
256229
target_include_directories(swigfaiss_sve PRIVATE ${PROJECT_SOURCE_DIR}/../..)
257-
target_include_directories(faiss_example_external_module PRIVATE ${PROJECT_SOURCE_DIR}/../..)
258230

259231
find_package(Python REQUIRED
260232
COMPONENTS Development NumPy
@@ -280,7 +252,6 @@ target_link_libraries(swigfaiss PRIVATE faiss_python_callbacks)
280252
target_link_libraries(swigfaiss_avx2 PRIVATE faiss_python_callbacks)
281253
target_link_libraries(swigfaiss_avx512 PRIVATE faiss_python_callbacks)
282254
target_link_libraries(swigfaiss_sve PRIVATE faiss_python_callbacks)
283-
target_link_libraries(faiss_example_external_module PRIVATE faiss_python_callbacks)
284255

285256
configure_file(setup.py setup.py COPYONLY)
286257
configure_file(__init__.py __init__.py COPYONLY)

faiss/python/faiss_example_external_module.swig

-141
This file was deleted.

faiss/python/setup.py

+20-38
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,10 @@
44
# LICENSE file in the root directory of this source tree.
55

66
from __future__ import print_function
7-
7+
from setuptools import setup, find_packages
88
import os
9-
import platform
109
import shutil
11-
12-
from setuptools import find_packages, setup
10+
import platform
1311

1412
# make the faiss python package dir
1513
shutil.rmtree("faiss", ignore_errors=True)
@@ -22,32 +20,25 @@
2220
shutil.copyfile("extra_wrappers.py", "faiss/extra_wrappers.py")
2321
shutil.copyfile("array_conversions.py", "faiss/array_conversions.py")
2422

25-
ext = ".pyd" if platform.system() == "Windows" else ".so"
26-
prefix = "Release/" * (platform.system() == "Windows")
23+
ext = ".pyd" if platform.system() == 'Windows' else ".so"
24+
prefix = "Release/" * (platform.system() == 'Windows')
2725

2826
swigfaiss_generic_lib = f"{prefix}_swigfaiss{ext}"
2927
swigfaiss_avx2_lib = f"{prefix}_swigfaiss_avx2{ext}"
3028
swigfaiss_avx512_lib = f"{prefix}_swigfaiss_avx512{ext}"
3129
callbacks_lib = f"{prefix}libfaiss_python_callbacks{ext}"
3230
swigfaiss_sve_lib = f"{prefix}_swigfaiss_sve{ext}"
33-
faiss_example_external_module_lib = f"_faiss_example_external_module{ext}"
3431

3532
found_swigfaiss_generic = os.path.exists(swigfaiss_generic_lib)
3633
found_swigfaiss_avx2 = os.path.exists(swigfaiss_avx2_lib)
3734
found_swigfaiss_avx512 = os.path.exists(swigfaiss_avx512_lib)
3835
found_callbacks = os.path.exists(callbacks_lib)
3936
found_swigfaiss_sve = os.path.exists(swigfaiss_sve_lib)
40-
found_faiss_example_external_module_lib = os.path.exists(
41-
faiss_example_external_module_lib
42-
)
4337

44-
assert (
45-
found_swigfaiss_generic or found_swigfaiss_avx2 or found_swigfaiss_avx512 or found_swigfaiss_sve or found_faiss_example_external_module_lib
46-
), (
47-
f"Could not find {swigfaiss_generic_lib} or "
48-
f"{swigfaiss_avx2_lib} or {swigfaiss_avx512_lib} or {swigfaiss_sve_lib} or {faiss_example_external_module_lib}. "
38+
assert (found_swigfaiss_generic or found_swigfaiss_avx2 or found_swigfaiss_avx512 or found_swigfaiss_sve), \
39+
f"Could not find {swigfaiss_generic_lib} or " \
40+
f"{swigfaiss_avx2_lib} or {swigfaiss_avx512_lib} or {swigfaiss_sve_lib}. " \
4941
f"Faiss may not be compiled yet."
50-
)
5142

5243
if found_swigfaiss_generic:
5344
print(f"Copying {swigfaiss_generic_lib}")
@@ -73,17 +64,7 @@
7364
shutil.copyfile("swigfaiss_sve.py", "faiss/swigfaiss_sve.py")
7465
shutil.copyfile(swigfaiss_sve_lib, f"faiss/_swigfaiss_sve{ext}")
7566

76-
if found_faiss_example_external_module_lib:
77-
print(f"Copying {faiss_example_external_module_lib}")
78-
shutil.copyfile(
79-
"faiss_example_external_module.py", "faiss/faiss_example_external_module.py"
80-
)
81-
shutil.copyfile(
82-
faiss_example_external_module_lib,
83-
f"faiss/_faiss_example_external_module{ext}",
84-
)
85-
86-
long_description = """
67+
long_description="""
8768
Faiss is a library for efficient similarity search and clustering of dense
8869
vectors. It contains algorithms that search in sets of vectors of any size,
8970
up to ones that possibly do not fit in RAM. It also contains supporting
@@ -92,19 +73,20 @@
9273
are implemented on the GPU. It is developed by Facebook AI Research.
9374
"""
9475
setup(
95-
name="faiss",
96-
version="1.9.0",
97-
description="A library for efficient similarity search and clustering of dense vectors",
76+
name='faiss',
77+
version='1.9.0',
78+
description='A library for efficient similarity search and clustering of dense vectors',
9879
long_description=long_description,
99-
url="https://github.com/facebookresearch/faiss",
100-
author="Matthijs Douze, Jeff Johnson, Herve Jegou, Lucas Hosseini",
101-
author_email="matthijs@meta.com",
102-
license="MIT",
103-
keywords="search nearest neighbors",
104-
install_requires=["numpy", "packaging"],
105-
packages=["faiss", "faiss.contrib", "faiss.contrib.torch"],
80+
url='https://github.com/facebookresearch/faiss',
81+
author='Matthijs Douze, Jeff Johnson, Herve Jegou, Lucas Hosseini',
82+
author_email='matthijs@meta.com',
83+
license='MIT',
84+
keywords='search nearest neighbors',
85+
86+
install_requires=['numpy', 'packaging'],
87+
packages=['faiss', 'faiss.contrib', 'faiss.contrib.torch'],
10688
package_data={
107-
"faiss": ["*.so", "*.pyd"],
89+
'faiss': ['*.so', '*.pyd'],
10890
},
10991
zip_safe=False,
11092
)

0 commit comments

Comments
 (0)