ci(datasets): Accelerate CI using uv

.github/workflows/e2e-tests.yml

@@ -29,13 +29,16 @@ jobs:
           path: ~/.cache/pip
           key: ${{inputs.plugin}}-${{inputs.os}}-python-${{inputs.python-version}}
           restore-keys: ${{inputs.plugin}}
+      - name: Install uv
+        run: |
+          python -m pip install "uv==0.1.13"
       - name: Install dependencies
         run: |
           cd ${{ inputs.plugin }}
-          pip install git+https://github.com/kedro-org/kedro@main
-          pip install ".[test]"
+          uv pip install --system "kedro @ git+https://github.com/kedro-org/kedro@main"
+          uv pip install --system "${{inputs.plugin}}[test] @ ."
       - name: pip freeze
-        run: pip freeze
+        run: uv pip freeze --system
       - name: Run end to end tests
         # Custom shell to run kedro-docker e2e-tests because -it flag for `docker run`
         # isn't supported on Github Actions. See https://github.com/actions/runner/issues/241

.github/workflows/kedro-datasets.yml

@@ -36,7 +36,7 @@ jobs:
       os: ubuntu-latest
       python-version: "3.11"
 
-  RTD-build:
+  check-docs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -51,11 +51,13 @@ jobs:
           path: ~/.cache/pip
           key: kedro-datasets-ubuntu-latest-python-"3.9"
           restore-keys: kedro-datasets
+      - name: Install uv
+        run: |
+          python -m pip install "uv==0.1.13"
       - name: Install dependencies
         run: |
-          python -m pip install -U "pip>=21.2,<23.2" # Temporary fix
           cd kedro-datasets
-          pip install ".[docs,test]"
-      - name: RTD build for kedro-datasets
+          uv pip install --system "kedro-datasets[docs,test] @ ."
+      - name: Documentation check for kedro-datasets
         run: |
-          make rtd
+          make check-datasets-docs

.github/workflows/lint.yml

@@ -29,13 +29,15 @@ jobs:
           path: ~/.cache/pip
           key: ${{inputs.plugin}}-${{inputs.os}}-python-${{inputs.python-version}}
           restore-keys: ${{inputs.plugin}}
+      - name: Install uv
+        run: |
+          python -m pip install "uv==0.1.13"
       - name: Install dependencies
         run: |
             cd ${{ inputs.plugin }}
-            python -m pip install -U "pip>=21.2,<23.2" # Temporary fix
-            pip install git+https://github.com/kedro-org/kedro@main
-            pip install ".[test]"
-            pip freeze
+            uv pip install --system "kedro @ git+https://github.com/kedro-org/kedro@main"
+            uv pip install --system "${{inputs.plugin}}[test] @ ."
+            uv pip freeze --system
       - name: Install pre-commit hooks
         run: |
             pre-commit install --install-hooks

.github/workflows/unit-tests.yml

@@ -38,18 +38,19 @@ jobs:
           path: ~\AppData\Local\pip\Cache
           key: ${{inputs.plugin}}-${{inputs.os}}-python-${{inputs.python-version}}
           restore-keys: ${{inputs.plugin}}
-      - name: Install Kedro
-        run: pip install git+https://github.com/kedro-org/kedro@main
       - name: Add MSBuild to PATH
         if: inputs.os == 'windows-latest'
         uses: microsoft/setup-msbuild@v2
+      - name: Install uv
+        run: |
+          python -m pip install "uv==0.1.13"
       - name: Install dependencies
         run: |
           cd ${{ inputs.plugin }}
-          python -m pip install -U "pip>=21.2,<23.2" # Temporary fix
-          pip install ".[test]"
+          uv pip install --system "kedro @ git+https://github.com/kedro-org/kedro@main"
+          uv pip install --system "${{inputs.plugin}}[test] @ ."
       - name: pip freeze
-        run: pip freeze
+        run: uv pip freeze --system
       - name: Run unit tests for Linux / kedro-airflow, kedro-docker, kedro-telemetry
         if: inputs.os != 'windows-latest' && inputs.plugin != 'kedro-datasets'
         run: make plugin=${{ inputs.plugin }} test

Makefile

@@ -87,5 +87,5 @@ test-snowflake-only:
 	cd kedro-datasets && pytest --no-cov --numprocesses 1 --dist loadfile -m snowflake
 	cd kedro-datasets && pytest kedro_datasets/snowflake --doctest-modules --doctest-continue-on-failure --no-cov
 
-rtd:
+check-datasets-docs:
 	cd kedro-datasets && python -m sphinx -WETan -j auto -D language=en -b linkcheck -d _build/doctrees docs/source _build/linkcheck

kedro-datasets/kedro_datasets/huggingface/hugging_face_dataset.py

@@ -23,12 +23,16 @@ class HFDataset(AbstractVersionedDataset):
 
     .. code-block:: pycon
 
+       >>> from datasets.utils.logging import disable_progress_bar, set_verbosity, ERROR
+       >>> disable_progress_bar()  # for doctest to pass
+       >>> set_verbosity(ERROR)  # for doctest to pass
        >>> from kedro_datasets.huggingface import HFDataset
-       >>> dataset = HFDataset(dataset_name="yelp_review_full")
-       >>> yelp_review_full = dataset.load()
-       >>> assert "train" in yelp_review_full
-       >>> assert "test" in yelp_review_full
-       >>> assert len(yelp_review_full["train"]) == 650000
+       >>> dataset = HFDataset(dataset_name="openai_humaneval")
+       >>> ds = dataset.load()  # doctest: +ELLIPSIS
+       Downloading and preparing dataset ...
+       Dataset ...
+       >>> assert "test" in ds
+       >>> assert len(ds["test"]) == 164
 
     """
 

kedro-datasets/pyproject.toml

@@ -56,7 +56,7 @@ matlab-matlabdataset = ["scipy"]
 matlab = ["kedro-datasets[matlab-matlabdataset]"]
 
 matplotlib-matplotlibwriter = ["matplotlib>=3.0.3, <4.0"]
-matplotlib = ["kedro-datasets[]"]
+matplotlib = ["kedro-datasets[matplotlib-matplotlibwriter]"]
 
 netcdf = ["kedro-datasets[netcdf-netcdfdataset]"]
 netcdf-netcdfdataset = ["h5netcdf>=1.2.0","netcdf4>=1.6.4","xarray>=2023.1.0"]