diff --git a/.github/workflows/build_pex.yml b/.github/workflows/build_pex.yml index f7b84a5a524..e0b57257cec 100644 --- a/.github/workflows/build_pex.yml +++ b/.github/workflows/build_pex.yml @@ -14,15 +14,13 @@ on: jobs: build_pex: name: Build PEX - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: python:3.6-buster outputs: pex-file-name: ${{ steps.get-pex-filename.outputs.pex-file-name }} steps: - uses: actions/checkout@v4 - - name: Set up Python 3.6 - uses: actions/setup-python@v5 - with: - python-version: 3.6 - uses: actions/cache@v4 with: path: ~/.cache/pip diff --git a/.github/workflows/build_whl.yml b/.github/workflows/build_whl.yml index be1b76fc4ad..f62ccc9e462 100644 --- a/.github/workflows/build_whl.yml +++ b/.github/workflows/build_whl.yml @@ -13,27 +13,27 @@ on: jobs: build_whl: name: Build WHL - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: python:3.6-buster outputs: whl-file-name: ${{ steps.get-whl-filename.outputs.whl-file-name }} tar-file-name: ${{ steps.get-tar-filename.outputs.tar-file-name }} steps: + - name: Install Ubuntu dependencies + run: | + apt-get -y -qq update + apt-get install -y gettext sudo sqlite3 binutils-aarch64-linux-gnu binutils-arm-linux-gnueabihf git-lfs - uses: actions/checkout@v4 with: fetch-depth: 0 lfs: true - - name: Install Ubuntu dependencies - run: | - sudo apt-get -y -qq update - sudo apt-get install -y gettext sudo sqlite3 binutils-aarch64-linux-gnu binutils-arm-linux-gnueabihf - - name: Set up Python 3.6 - uses: actions/setup-python@v5 - with: - python-version: 3.6 - name: Use Node.js uses: actions/setup-node@v4 with: node-version: '18.x' + - name: Install Yarn + run: npm install -g yarn@1 - name: Get yarn cache directory path id: yarn-cache-dir-path run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT diff --git a/.github/workflows/c_extensions.yml b/.github/workflows/c_extensions.yml deleted file mode 100644 index 1534f3a690b..00000000000 --- a/.github/workflows/c_extensions.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: Static dependencies with and without C extensions - -on: - push: - branches: - - develop - - 'release-v**' - pull_request: - branches: - - develop - - 'release-v**' - -jobs: - pre_job: - name: Path match check - runs-on: ubuntu-latest - # Map a step output to a job output - outputs: - should_skip: ${{ steps.skip_check.outputs.should_skip }} - steps: - - id: skip_check - uses: fkirc/skip-duplicate-actions@master - with: - github_token: ${{ github.token }} - paths: '["**.py", "requirements/base.txt", "requirements/build.txt"]' - c_ext: - name: C Extensions - needs: pre_job - if: ${{ needs.pre_job.outputs.should_skip != 'true' }} - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.6 - - name: pip cache - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-ext-${{ hashFiles('requirements/*.txt') }} - restore-keys: | - ${{ runner.os }}-pip-ext - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements/base.txt - pip install -r requirements/build.txt - pip install -r requirements/test.txt - - name: Check C extensions build and run - run: | - # Ensure that for this Python version, we can actually compile ALL files - # in the kolibri directory - python -m compileall -q kolibri -x py2only - # Until we have staged builds, we will be running this in each and every - # environment even though builds should be done in Py 3.6 - make staticdeps - make staticdeps-cext - pip install . - # Start and stop kolibri - kolibri start --port=8081 - kolibri stop - # Run just tests in test/ - py.test --color=no test/ - no_c_ext: - name: No C Extensions - needs: pre_job - if: ${{ needs.pre_job.outputs.should_skip != 'true' }} - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.6 - - name: pip cache - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-no-ext-${{ hashFiles('requirements/*.txt') }} - restore-keys: | - ${{ runner.os }}-pip-no-ext - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements/base.txt - pip install -r requirements/build.txt - pip install -r requirements/test.txt - - name: Check No C extensions build and run - run: | - # Ensure that for this Python version, we can actually compile ALL files - # in the kolibri directory - python -m compileall -q kolibri -x py2only - # Until we have staged builds, we will be running this in each and every - # environment even though builds should be done in Py 3.6 - make staticdeps - pip install . - # Start and stop kolibri - kolibri start --port=8081 - kolibri stop - # Run just tests in test/ - py.test --color=no test/ diff --git a/.github/workflows/morango_integration.yml b/.github/workflows/morango_integration.yml index c49eb6faf1e..2a4d4f2a910 100644 --- a/.github/workflows/morango_integration.yml +++ b/.github/workflows/morango_integration.yml @@ -22,15 +22,15 @@ jobs: github_token: ${{ github.token }} paths: '["requirements/base.txt", ".github/workflows/morango_integration.yml"]' morango_integration_tests_sqlite: - name: Morango Integration Tests + name: Morango Integration Tests for currently supported Python versions needs: pre_job if: needs.pre_job.outputs.should_skip != 'true' - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest env: INTEGRATION_TEST: 'true' strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12'] + python-version: [3.9, '3.10', '3.11', '3.12'] steps: - name: Checkout repository @@ -53,6 +53,34 @@ jobs: - name: Test with tox run: tox -e py${{ matrix.python-version }} -- kolibri/core/auth/test/test_morango_integration.py + morango_integration_tests_sqlite_eol_python: + name: Morango Integration Tests for EOL Python versions + needs: pre_job + if: needs.pre_job.outputs.should_skip != 'true' + runs-on: ubuntu-latest + env: + INTEGRATION_TEST: 'true' + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + container: + image: python:${{ matrix.python-version }}-buster + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install tox + run: | + python -m pip install --upgrade pip + pip install "tox<4" + - name: tox env cache + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.tox/py${{ matrix.python-version }} + key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-${{ hashFiles('requirements/*.txt') }} + - name: Test with tox + run: tox -e py${{ matrix.python-version }} -- kolibri/core/auth/test/test_morango_integration.py + morango_integration_tests_postgres: name: Morango Integration Tests with PostgreSQL diff --git a/.github/workflows/pr_build_kolibri.yml b/.github/workflows/pr_build_kolibri.yml index 1e9cbdb256f..019a513dcf2 100644 --- a/.github/workflows/pr_build_kolibri.yml +++ b/.github/workflows/pr_build_kolibri.yml @@ -66,3 +66,47 @@ jobs: with: deb-file-name: ${{ needs.deb.outputs.deb-file-name }} ref: v1.0.0 + test_whl_file: + name: Test WHL file + needs: whl + strategy: + matrix: + python-version: [3.9, '3.10', '3.11', '3.12'] + cext: [true, false] + runs-on: ubuntu-latest + steps: + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/download-artifact@v4 + with: + name: ${{ needs.whl.outputs.whl-file-name }} + path: dist + - name: Install WHL file + run: pip install dist/${{ needs.whl.outputs.whl-file-name }} + - name: Smoke test WHL file + env: + KOLIBRI_NO_C_EXTENSIONS: ${{ matrix.cext == false && '1' || '' }} + run: kolibri configure setup + test_whl_file_eol_python: + name: Test WHL file with EOL Python versions + needs: whl + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + cext: [true, false] + runs-on: ubuntu-latest + container: + image: python:${{ matrix.python-version }}-buster + steps: + - uses: actions/download-artifact@v4 + with: + name: ${{ needs.whl.outputs.whl-file-name }} + path: dist + - name: Install WHL file + run: pip install dist/${{ needs.whl.outputs.whl-file-name }} + - name: Smoke test WHL file + env: + KOLIBRI_NO_C_EXTENSIONS: ${{ matrix.cext == false && '1' || '' }} + run: kolibri configure setup diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 3ef0687ee84..d6386f15eb6 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -25,7 +25,7 @@ on: jobs: deploy: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index e5223672819..dfc8c700344 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -24,13 +24,13 @@ jobs: github_token: ${{ github.token }} paths: '["**.py", "requirements/*.txt", ".github/workflows/tox.yml", "tox.ini"]' unit_test: - name: Python unit tests + name: Python unit tests for currently supported Python versions needs: pre_job - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: max-parallel: 5 matrix: - python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12'] + python-version: [3.9, '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v4 @@ -53,6 +53,32 @@ jobs: - name: Test with tox if: ${{ needs.pre_job.outputs.should_skip != 'true' }} run: tox -e py${{ matrix.python-version }} + unit_test_eol_python: + name: Python unit tests for EOL Python versions + needs: pre_job + runs-on: ubuntu-latest + strategy: + max-parallel: 5 + matrix: + python-version: [3.6, 3.7, 3.8] + container: + image: python:${{ matrix.python-version }}-buster + steps: + - uses: actions/checkout@v4 + - name: Install tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: | + python -m pip install --upgrade pip + pip install "tox<4" + - name: tox env cache + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.tox/py${{ matrix.python-version }} + key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-${{ hashFiles('requirements/*.txt') }} + - name: Test with tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: tox -e py${{ matrix.python-version }} postgres: name: Python postgres unit tests needs: pre_job diff --git a/kolibri/utils/env.py b/kolibri/utils/env.py index 154080db30b..e117220a977 100644 --- a/kolibri/utils/env.py +++ b/kolibri/utils/env.py @@ -28,6 +28,9 @@ def settings_module(): "KOLIBRI_APK_VERSION_NAME": { "description": "Version name for the Kolibri APK (Android Installer)", }, + "KOLIBRI_NO_C_EXTENSIONS": { + "description": "Disable C extensions.", + }, "LISTEN_PID": { "description": """ The PID of the process to listen for signals from - @@ -136,6 +139,7 @@ def set_env(): from the distributed version in case it exists before importing anything else. """ + monkey_patch_markdown() monkey_patch_distutils() @@ -143,8 +147,9 @@ def set_env(): sys.path = [os.path.realpath(os.path.dirname(kolibri_dist.__file__))] + sys.path - # Add path for c extensions to sys.path - prepend_cext_path(os.path.realpath(os.path.dirname(kolibri_dist.__file__))) + if not os.environ.get("KOLIBRI_NO_C_EXTENSIONS", False): + # Add path for c extensions to sys.path + prepend_cext_path(os.path.realpath(os.path.dirname(kolibri_dist.__file__))) # Set default env for key, value in ENVIRONMENT_VARIABLES.items(): diff --git a/kolibri/utils/tests/test_sanity_check.py b/kolibri/utils/tests/test_sanity_check.py index d2aa72e8a08..b288d814e22 100644 --- a/kolibri/utils/tests/test_sanity_check.py +++ b/kolibri/utils/tests/test_sanity_check.py @@ -15,7 +15,9 @@ class SanityCheckTestCase(TestCase): @patch("kolibri.utils.sanity_checks.logging.error") @override_option( - "Paths", "CONTENT_DIR", "Z:\\NOTREAL" if sys.platform == "win32" else "/dir_dne" + "Paths", + "CONTENT_DIR", + "Z:\\NOTREAL" if sys.platform == "win32" else "/proc/self/mem/impossible", ) def test_content_dir_dne(self, logging_mock): with self.assertRaises(SystemExit):