Merge pull request #7 from sqrl-planner/feat/ttb-migration

feat: uoft timetable api (ttb) integration

.flake8

@@ -1,4 +1,4 @@
 [flake8]
 ignore=E501
-exclude = tests/*
+exclude = tests/* src/gator/_vendor/*
 max-complexity = 10

.github/workflows/publish.yml

@@ -19,6 +19,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install poetry
+        poetry install
+        poetry run python-vendorize
     - name: Build and publish
       run: |
         poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }}

.github/workflows/pytest.yml

@@ -20,6 +20,7 @@ jobs:
         run: |
           pip install poetry
           poetry install
+          poetry run python-vendorize
       - name: Test with pytest
         run: |
           poetry run pytest

.gitignore

@@ -379,3 +379,6 @@ cython_debug/
 
 # VS Code
 .vscode/*
+
+# Vendor
+src/gator/_vendor/

conftest.py

@@ -0,0 +1,33 @@
+"""Configure the test suite."""
+from doctest import ELLIPSIS
+
+import mongomock
+import pytest
+from mongoengine import connect
+from sybil import Sybil
+from sybil.parsers.rest import DocTestParser, PythonCodeBlockParser
+
+# The name of the test database. This is used by the `mongo_client` fixture.
+TEST_DB = 'mongotest'
+
+
+pytest_collect_file = Sybil(
+    parsers=[
+        DocTestParser(optionflags=ELLIPSIS),
+        PythonCodeBlockParser(),
+    ],
+    patterns=['*.rst', '*.py'],
+).pytest()
+
+
+@pytest.fixture(autouse=True, scope='module')
+def mongo_client(request: pytest.FixtureRequest) -> mongomock.MongoClient:
+    """Return a MongoDB client for testing."""
+    # Explicitly set the UUID representation to standard, since mongomock
+    # defaults to Python's legacy representation which will raise a warning
+    connection = connect(db=TEST_DB, host='mongomock://localhost',
+                         uuidRepresentation='standard')
+    # Add a handler to drop the test database after the test is complete
+    request.addfinalizer(lambda: connection.drop_database(TEST_DB))
+
+    return connection

pyproject.toml

@@ -1,54 +1,61 @@
-[tool.poetry]
-name = "gator-core"
-version = "0.0.0"  # placeholder for the real version, which is retrieved from git tags
-description = "A dataset aggregation framework for Sqrl Planner."
-authors = ["Shon Verch <verchshon@gmail.com>"]
-license = "MIT"
-readme = "README.md"
-
-packages = [
-    { include = "gator", from = "src" }
-]
-include = [
-    { path = "tests", format = "sdist" }
-]
-exclude = [
-    "**/*.pyc",
-    "**/*.pyi",
-]
-
-[tool.poetry.dependencies]
-python = "^3.9"
-mongoengine = ">=0.20"
-marshmallow = "^3.17.0"
-marshmallow-enum = "^1.5.1"
-requests = "^2.28.2"
-bs4 = "^0.0.1"
-routes = "^2.5.1"
-
-[tool.poetry.dev-dependencies]
-pre-commit = "^2.20.0"
-pytest = "^7.1.2"
-pytest-cov = "^3.0.0"
-pytest-mock = "^3.7.0"
-pytest-httpserver = "^1.0.4"
-
-[tool.poetry-dynamic-versioning]
-enable = true
-
-[build-system]
-requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"]
-build-backend = "poetry.core.masonry.api"
-
-[tool.pyright]
-include = ["gator/**", "tests/**"]
-exclude = ["**/__pycache__"]
-
-typeCheckingMode = "basic"
-pythonVersion = "3.9"
-pythonPlatform = "All"
-typeshedPath = "typeshed"
-enableTypeIgnoreComments = true
-
-# This is required as the CI pre-commit does not download required modules
-reportMissingImports = "none"
+[tool.poetry]
+name = "gator-core"
+version = "0.0.0"  # placeholder for the real version, which is retrieved from git tags
+description = "A dataset aggregation framework for Sqrl Planner."
+authors = ["Shon Verch <verchshon@gmail.com>"]
+license = "MIT"
+readme = "README.md"
+
+packages = [
+    { include = "gator/**/*", from = "src" }
+]
+include = [
+    { path = "src/gator/_vendor/**/*", format = ["sdist", "wheel"] },
+    { path = "tests", format = "sdist" }
+]
+exclude = [
+    "**/*.pyc",
+    "**/*.pyi",
+]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+mongoengine = ">=0.20"
+marshmallow = "^3.17.0"
+requests = "^2.28.2"
+bs4 = "^0.0.1"
+routes = "^2.5.1"
+
+[tool.poetry.dev-dependencies]
+pre-commit = "^2.20.0"
+pytest = "^7.1.2"
+pytest-cov = "^3.0.0"
+pytest-mock = "^3.7.0"
+pytest-httpserver = "^1.0.4"
+
+[tool.poetry.group.dev.dependencies]
+mypy = "^1.0.1"
+mongo-types = "^0.15.1"
+vendorize = "^0.3.0"
+mongomock = "^4.1.2"
+sybil = "^5.0.0"
+
+[tool.poetry-dynamic-versioning]
+enable = true
+
+[build-system]
+requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.pyright]
+include = ["gator/**", "tests/**"]
+exclude = ["**/__pycache__", "src/gator/_vendor/**"]
+
+typeCheckingMode = "basic"
+pythonVersion = "3.9"
+pythonPlatform = "All"
+typeshedPath = "typeshed"
+enableTypeIgnoreComments = true
+
+# This is required as the CI pre-commit does not download required modules
+reportMissingImports = "none"

pytest.ini

@@ -1,3 +1,3 @@
-# content of pytest.ini
-[pytest]
-addopts = --doctest-modules
+# content of pytest.ini
+[pytest]
+addopts = -p no:doctest

src/gator/core/data/dataset.py

@@ -1,19 +1,23 @@
 # type: ignore
-"""Base classes for a datasets."""
+"""Base classes for datasets."""
 import fnmatch
 from abc import ABC, abstractclassmethod, abstractmethod, abstractproperty
 from typing import Any, Iterator, Optional, Type, Union
 
-from gator.core.models.common import Record
+from mongoengine import Document
+
 from gator.core.models.timetable import Session
 
 
 class Dataset(ABC):
-    """A dataset that returns :class:`gator.models.Record` instances.
-
-    All subclasses should implement the `get` method to return a list of
-    records. The `slug`, `name`, and `description` properties should also be
-    implemented to provide metadata about the dataset.
+    """A dataset that converts free-form data into mongoengine documents.
+
+    All subclasses should implement functionality for a) pulling data from a
+    source (e.g. a database, a file, or an API) and returning it as a list of
+    `(id, data)` records via the `get` method, and b) processing each record
+    into a :class:`mongoengine.Document` object via the `process` method.
+    The `slug`, `name`, and `description` properties should also be implemented
+    to provide metadata about the dataset.
     """
 
     @abstractproperty
@@ -32,45 +36,72 @@ def description(self) -> str:
         raise NotImplementedError()
 
     @abstractmethod
-    def get(self) -> Iterator[Record]:
-        """Return an iterator that lazily yields records from this dataset."""
+    def get(self) -> Iterator[tuple[str, Any]]:
+        """Return an iterator that lazily yields `(id, data)` records.
+
+        The `id` should be a unique identifier for the record, and the `data`
+        can be any hashable object. A hash of the `data` object will be compared
+        with a hash stored in the database for the record with the given `id`.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def process(self, id: str, data: Any) -> Document:
+        """Process the given record into a :class:`mongoengine.Document`.
+
+        Args:
+            id: The unique identifier for the record.
+            data: The data for the record.
+        """
         raise NotImplementedError()
 
 
 class SessionalDataset(Dataset):
-    """A dataset that is specific to a session.
+    """A dataset that is specific to one or more sessions.
 
-    For example, a dataset of courses offered in a particular session, or a
-    dataset of clubs that are active in a particular session.
+    For example, a dataset of courses offered in a set of particular sessions,
+    or a dataset of clubs that are active in a single session.
 
-    All subclasses must implement the `_get_latest_session` method to return
-    the most up-to-date session for this dataset.
+    All subclasses must implement the `_get_latest_sessions` method to return
+    the most up-to-date sessions for this dataset.
 
     Instance Attributes:
-        session: The session that this dataset is specific to.
+        sessions: The sessions that this dataset is specific to.
     """
 
-    session: Session
+    sessions: list[Session]
 
-    def __init__(self, session: Optional[Union[Session, str]] = None) -> None:
+    def __init__(self,
+                 sessions: Optional[list[Union[str, Session]]] = None,
+                 session: Optional[Union[str, Session]] = None) -> None:
         """Initialize a SessionalDataset.
 
         Args:
-            session: An optional session that can be supplied instead of the
-                default. This can be an instance of Session or a string providing
-                the session code. If None, the latest session will be used.
+            sessions: The sessions that this dataset tracks. Each element can
+                either be a session code, which will be parsed into a Session
+                object, or a Session object itself. If no sessions are provided,
+                the most up-to-date sessions will be used.
+            session: Instead of providing a list of sessions, a single session
+                can be provided. This is equivalent to providing a list with a
+                single session. If both `sessions` and `session` are provided,
+                `sessions` will take precedence.
         """
         super().__init__()
-        if isinstance(session, str):
-            self.session = Session.parse(session)
-        elif session is None:
-            self.session = self._get_latest_session()
+        # Prefer the `sessions` argument. In the case it is None, use the
+        # `session` argument if it is provided, otherwise default to None.
+        sessions = sessions or ([session] if session is not None else None)
+        if sessions is None:
+            # Use the most up-to-date sessions
+            self.sessions = self._get_latest_sessions()
         else:
-            self.session = session
+            # Parse the sessions
+            self.sessions = [
+                Session.from_code(session) if not isinstance(session, Session) else session
+                for session in sessions]
 
     @abstractclassmethod
-    def _get_latest_session(cls) -> Session:
-        """Return the most up-to-date session for this dataset.
+    def _get_latest_sessions(cls) -> list[Session]:
+        """Return the most up-to-date sessions for this dataset.
 
         Raise a ValueError if the session could not be found.
         """

src/gator/core/data/utils/serialization.py

@@ -6,7 +6,7 @@ def nullable_convert(value: Any, func: Callable[[Any], Any]) -> Any:
     """Convert a value given a conversion function if it is not None.
 
     Remarks:
-        - This will silently handling None inputs by returning None.
+        - This will silently handle None inputs by returning None.
         - Useful for when a function depends on a non-None input.
 
     Args:

src/gator/core/models/__init__.py

@@ -1,3 +1 @@
-"""Models for the Gator application."""
-from gator.core.models import common  # noqa: F401
-from gator.core.models import timetable  # noqa: F401
+"""All models in Gator."""