Refactor pure vs cython builds

Author: titusz

.github/workflows/tests.yml

@@ -8,6 +8,7 @@ jobs:
       matrix:
         python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
         os: [ubuntu-20.04, macos-12, windows-2019]
+        cython: [false, true]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
@@ -20,27 +21,29 @@ jobs:
         with:
           poetry-version: 1.8.4
 
-      - name: Install Dependencies
+      - name: Install Dependencies (Pure Python)
+        if: ${{ !matrix.cython }}
         run: poetry install --without dev,docs
 
-      - name: Run Tests (Pure Python with coverage)
-        run: poetry run pytest --cov=iscc_core --cov-report=xml -q tests
+      - name: Install Dependencies (with Cython)
+        if: ${{ matrix.cython }}
+        run: poetry install --without dev,docs --extras cython
 
-      - name: Build Extension modules
-        run: poetry install --without dev,docs --extras turbo
-
-      - name: Run Tests (With Extension Modules)
-        run: poetry run pytest -q tests --turbo
+      - name: Run Tests
+        run: |
+          poetry run python -c "import iscc_core; print(f'Using Cython: {iscc_core.USING_CYTHON}')"
+          poetry run pytest -v tests
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
-        if: matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.7'
+        if: matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.9' && !matrix.cython
 
       - name: Build Wheel
+        if: ${{ matrix.cython }}
         run: poetry build -f wheel
 
       - name: Test Wheel Installation
-        if: runner.os != 'Windows'
+        if: ${{ matrix.cython && runner.os != 'Windows' }}
         run: |
           python -m venv venv
           source venv/bin/activate
@@ -49,7 +52,7 @@ jobs:
           python -c "import iscc_core; print(iscc_core.turbo())"
 
       - name: Test Wheel Installation on Windows
-        if: runner.os == 'Windows'
+        if: ${{ matrix.cython && runner.os == 'Windows' }}
         run: |
           python -m venv venv
           venv\Scripts\Activate.ps1
@@ -59,6 +62,7 @@ jobs:
           python -c "import iscc_core; print(iscc_core.turbo())"
 
       - name: Collect Wheel
+        if: ${{ matrix.cython }}
         uses: actions/upload-artifact@v3
         with:
           path: dist/*.whl

build.py

@@ -1,64 +1,32 @@
 # -*- coding: utf-8 -*-
-"""
-Build cython extension modules.
+import os
+from setuptools import Extension
 
-The shared library can also be built manually using the command:
-$ cythonize -X language_level=3 -a -i ./iscc_core/cdc.py
-$ cythonize -X language_level=3 -a -i ./iscc_core/minhash.py
-$ cythonize -X language_level=3 -a -i ./iscc_core/simhash.py
-$ cythonize -X language_level=3 -a -i ./iscc_core/dct.py
-$ cythonize -X language_level=3 -a -i ./iscc_core/wtahash.py
-"""
-try:
-    from Cython.Build import cythonize, build_ext
-except ImportError:
-    # dummy build function for poetry
-    def build(setup_kwargs):
-        pass
 
-else:
+def build(setup_kwargs):
+    try:
+        from Cython.Build import cythonize
 
-    class build_ext_gracefull(build_ext):
-        def run(self):
-            try:
-                print("Trying to compile C accelerator modules")
-                super().run()
-                print("Successfully comiled C accelerator modules")
-            except Exception as e:
-                print(e)
-                print("********************************************************************")
-                print("Failed to compile C accelerator module, falling back to pure python.")
-                print("********************************************************************")
+        use_cython = True
+    except ImportError:
+        use_cython = False
 
-        def build_extensions(self):
-            try:
-                print("Trying to compile C accelerator modules")
-                super().build_extensions()
-                print("Successfully comiled C accelerator modules")
-            except Exception as e:
-                print(e)
-                print("********************************************************************")
-                print("Failed to compile C accelerator module, falling back to pure python.")
-                print("********************************************************************")
-
-    def build(setup_kwargs):
+    if use_cython:
         try:
-            setup_kwargs.update(
-                dict(
-                    ext_modules=cythonize(
-                        [
-                            "iscc_core/cdc.py",
-                            "iscc_core/minhash.py",
-                            "iscc_core/simhash.py",
-                            "iscc_core/dct.py",
-                            "iscc_core/wtahash.py",
-                        ]
-                    ),
-                    cmdclass=dict(build_ext=build_ext_gracefull),
-                )
+            ext_modules = cythonize(
+                [
+                    Extension("iscc_core.cdc", ["iscc_core/cdc.py"]),
+                    Extension("iscc_core.minhash", ["iscc_core/minhash.py"]),
+                    Extension("iscc_core.simhash", ["iscc_core/simhash.py"]),
+                    Extension("iscc_core.dct", ["iscc_core/dct.py"]),
+                    Extension("iscc_core.wtahash", ["iscc_core/wtahash.py"]),
+                ],
+                compiler_directives={"language_level": "3"},
             )
+            setup_kwargs.update({"ext_modules": ext_modules})
+            print("Cython modules prepared for compilation")
         except Exception as e:
-            print(e)
-            print("********************************************************************")
-            print("Failed to compile C accelerator module, falling back to pure python.")
-            print("********************************************************************")
+            print(f"Failed to prepare Cython modules: {e}")
+            print("Falling back to pure Python")
+    else:
+        print("Cython not available, using pure Python")

iscc_core/__init__.py

@@ -1,3 +1,16 @@
+import sys
+
+
+def _using_cython_modules():
+    modules = ["cdc", "minhash", "simhash", "dct", "wtahash"]
+    return any(
+        getattr(sys.modules.get(f"iscc_core.{module}"), "__file__", "").endswith((".so", ".pyd"))
+        for module in modules
+    )
+
+
+USING_CYTHON = _using_cython_modules()
+
 __version__ = "1.1.0"
 from iscc_core.options import core_opts, conformant_options
 

iscc_core/check.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 """Inspect lib environment/installation"""
-import inspect
-from loguru import logger as log
+from iscc_core import USING_CYTHON
 
 
 __all__ = ["turbo"]
@@ -10,12 +9,4 @@
 def turbo():  # pragma: no cover
     # type: () -> bool
     """Check whether all optional cython extensions have been compiled to native modules."""
-    from iscc_core import cdc, minhash, simhash, dct, wtahash
-
-    modules = (cdc, minhash, simhash, dct, wtahash)
-    for module in modules:
-        module_file = inspect.getfile(module)
-        log.debug(f"Module {module.__name__} file: {module_file}")
-        if module_file.endswith(".py") or module_file.endswith(".pyc"):
-            return False
-    return True
+    return USING_CYTHON

tests/test_check.py

@@ -2,8 +2,5 @@
 import iscc_core as ic
 
 
-def test_check_turbo(turbo):
-    if turbo is False:
-        assert ic.turbo() is False
-    else:
-        assert ic.turbo() is True
+def test_check_turbo():
+    assert ic.USING_CYTHON == ic.turbo()