Merge pull request #178 from rstudio/feat-manifest-compat

Feat manifest compat

pins/boards.py

@@ -53,6 +53,8 @@ def info(self, path):
 
 
 class BaseBoard:
+    reserved_pin_names = {"_pins.yaml"}
+
     def __init__(
         self,
         board: "str | Path",
@@ -174,8 +176,11 @@ def pin_list(self):
         This is a low-level function; use pin_search() to get more data about
         each pin in a convenient form.
         """
+
         full_paths = self.fs.ls(self.board)
-        return list(map(self.keep_final_path_component, full_paths))
+        pin_names = map(self.keep_final_path_component, full_paths)
+
+        return [name for name in pin_names if name not in self.reserved_pin_names]
 
     def pin_fetch(self, name: str, version: Optional[str] = None) -> Meta:
         meta = self.pin_meta(name, version)
@@ -513,6 +518,8 @@ def validate_pin_name(self, name: str) -> None:
 
         if "/" in name:
             raise ValueError(f"Invalid pin name: {name}")
+        elif name in self.reserved_pin_names:
+            raise ValueError(f"The pin name '{name}' is reserved for internal use.")
 
     def path_to_pin(self, name: str) -> str:
         self.validate_pin_name(name)

pins/meta.py

@@ -1,10 +1,10 @@
 from typing import ClassVar
-from dataclasses import dataclass, asdict, field
+from dataclasses import dataclass, asdict, field, fields, InitVar
 from pathlib import Path
 
 import yaml
 
-from typing import Mapping, Union, Sequence, Optional
+from typing import Mapping, Union, Sequence, Optional, List
 
 from .versions import VersionRaw, Version, guess_version
 from ._types import StrOrFile, IOBase
@@ -41,6 +41,8 @@ class Meta:
         A title for the pin.
     description:
         A detailed description of the pin contents.
+    tags:
+        Optional tags applied to the pin.
     created:
         Datetime the pin was created (TODO: document format).
     pin_hash:
@@ -63,6 +65,8 @@ class Meta:
 
     """
 
+    _excluded: ClassVar["set[str]"] = {"name", "version", "local"}
+
     title: Optional[str]
     description: Optional[str]
 
@@ -82,10 +86,24 @@ class Meta:
     # pin_hash, created, etc.."
     version: VersionRaw
 
+    tags: Optional[List[str]] = None
     name: Optional[str] = None
     user: Mapping = field(default_factory=dict)
     local: Mapping = field(default_factory=dict)
 
+    unknown_fields: InitVar["dict | None"] = None
+
+    def __post_init__(self, unknown_fields: "dict | None"):
+        unknown_fields = {} if unknown_fields is None else unknown_fields
+
+        self._unknown_fields = unknown_fields
+
+    def __getattr__(self, k):
+        try:
+            return self._unknown_fields[k]
+        except KeyError:
+            raise AttributeError(f"No metadata field not found: {k}")
+
     def to_dict(self) -> Mapping:
         data = asdict(self)
 
@@ -94,21 +112,37 @@ def to_dict(self) -> Mapping:
     def to_pin_dict(self):
         d = self.to_dict()
 
-        del d["name"]
-        del d["version"]
-        del d["local"]
+        for k in self._excluded:
+            del d[k]
+
+        # TODO: once tag writing is implemented, delete this line
+        del d["tags"]
 
         return d
 
     @classmethod
     def from_pin_dict(cls, data, pin_name, version, local=None) -> "Meta":
-
         # TODO: re-arrange Meta argument positions to reflect what's been
         # learned about default arguments. e.g. title was not used at some
         # point in api_version 1
+        all_field_names = {entry.name for entry in fields(Meta)}
+
+        keep_fields = all_field_names - cls._excluded
+
         extra = {"title": None} if "title" not in data else {}
         local = {} if local is None else local
-        return cls(**data, **extra, name=pin_name, version=version, local=local)
+
+        meta_data = {k: v for k, v in data.items() if k in keep_fields}
+        unknown = {k: v for k, v in data.items() if k not in keep_fields}
+
+        return cls(
+            **meta_data,
+            **extra,
+            name=pin_name,
+            version=version,
+            local=local,
+            unknown_fields=unknown,
+        )
 
     def to_pin_yaml(self, f: Optional[IOBase] = None) -> "str | None":
         data = self.to_pin_dict()

pins/tests/conftest.py

@@ -12,6 +12,7 @@
 PATH_TO_EXAMPLE_VERSION = PATH_TO_EXAMPLE_BOARD / "df_csv/20220214T163720Z-9bfad/"
 EXAMPLE_PIN_NAME = "df_csv"
 
+PATH_TO_MANIFEST_BOARD = files("pins") / "tests/pin-board"
 
 # Based on https://github.com/machow/siuba/blob/main/siuba/tests/helpers.py
 BACKEND_MARKS = ["fs_s3", "fs_file", "fs_gcs", "fs_abfs", "fs_rsc"]

pins/tests/pin-board/_pins.yaml

@@ -0,0 +1,5 @@
+x:
+- x/20221215T180351Z-c3943/
+'y':
+- y/20221215T180357Z-9ae7a/
+- y/20221215T180400Z-b81d5/

pins/tests/pin-board/x/20221215T180351Z-c3943/data.txt

@@ -0,0 +1,9 @@
+file: x.json
+file_size: 23
+pin_hash: c3943ca5a9aab2df
+type: json
+title: 'x: a pinned integer vector'
+description: ~
+tags: ~
+created: 20221215T180351Z
+api_version: 1.0

pins/tests/pin-board/x/20221215T180351Z-c3943/x.json

@@ -0,0 +1 @@
+[1,2,3,4,5,6,7,8,9,10]

pins/tests/pin-board/y/20221215T180357Z-9ae7a/data.txt

@@ -0,0 +1,9 @@
+file: y.rds
+file_size: 61
+pin_hash: 9ae7a970010c84e0
+type: rds
+title: 'y: a pinned integer vector'
+description: ~
+tags: ~
+created: 20221215T180357Z
+api_version: 1.0

pins/tests/pin-board/y/20221215T180400Z-b81d5/data.txt

@@ -0,0 +1,9 @@
+file: y.json
+file_size: 53
+pin_hash: b81d5bea9e760608
+type: json
+title: 'y: a pinned integer vector'
+description: ~
+tags: ~
+created: 20221215T180400Z
+api_version: 1.0

pins/tests/pin-board/y/20221215T180400Z-b81d5/y.json

@@ -0,0 +1 @@
+[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]

pins/tests/test_compat.py

@@ -4,7 +4,7 @@
 
 from pins.errors import PinsError
 from pins.tests.helpers import xfail_fs
-from pins.tests.conftest import PATH_TO_EXAMPLE_BOARD
+from pins.tests.conftest import PATH_TO_EXAMPLE_BOARD, PATH_TO_MANIFEST_BOARD
 
 
 NOT_A_PIN = "not_a_pin_abcdefg"
@@ -22,6 +22,20 @@ def board(backend):
     backend.teardown_board(board)
 
 
+@pytest.fixture(scope="session")
+def board_manifest(backend):
+    # skip on rsconnect, since it can't add a manifest and the pin names
+    # are too short for use to upload (rsc requires names > 3 characters)
+    if backend.fs_name == "rsc":
+        pytest.skip()
+
+    board = backend.create_tmp_board(str(PATH_TO_MANIFEST_BOARD.absolute()))
+
+    yield board
+
+    backend.teardown_board(board)
+
+
 # pin_list --------------------------------------------------------------------
 
 
@@ -145,3 +159,40 @@ def test_compat_pin_read_supported(board):
 
 
 # pin_write ----
+
+# manifest -----
+
+
+def test_board_pin_write_manifest_name_error(board_manifest):
+    if board_manifest.fs.protocol == "rsc":
+        pytest.skip()
+
+    with pytest.raises(ValueError) as exc_info:
+        board_manifest.pin_write([1], "_pins.yaml", type="json")
+
+    assert "name '_pins.yaml' is reserved for internal use." in exc_info.value.args[0]
+
+
+def test_board_manifest_pin_list_no_internal_name(board_manifest):
+    assert set(board_manifest.pin_list()) == {"x", "y"}
+
+
+def test_board_manifest_pin_exist_internal_name_errors(board_manifest):
+    with pytest.raises(ValueError) as exc_info:
+        board_manifest.pin_exists("_pins.yaml")
+
+    assert "reserved for internal use." in exc_info.value.args[0]
+
+
+def test_board_manifest_pin_read_internal_errors(board_manifest):
+    with pytest.raises(ValueError) as exc_info:
+        board_manifest.pin_read("_pins.yaml")
+
+    assert "reserved for internal use." in exc_info.value.args[0]
+
+
+def test_board_manifest_pin_search(board_manifest):
+    res = board_manifest.pin_search("x", as_df=False)
+
+    assert len(res) == 1
+    assert res[0].name == "x"

pins/tests/test_meta.py

@@ -1,5 +1,6 @@
 import pytest
 import tempfile
+import yaml
 
 from datetime import datetime
 from io import StringIO
@@ -37,6 +38,18 @@ def test_meta_to_pin_dict_roundtrip(meta):
     assert meta == meta2
 
 
+def test_meta_unknown_fields():
+    m = Meta(**META_DEFAULTS, unknown_fields={"some_other_field": 1})
+
+    assert m.some_other_field == 1
+
+    with pytest.raises(AttributeError):
+        m.should_not_exist_here
+
+    assert "unknown_fields" not in m.to_pin_dict()
+    assert "some_other_field" not in m.to_pin_dict()
+
+
 def test_meta_factory_create():
     mf = MetaFactory()
     with tempfile.TemporaryDirectory() as tmp_dir:
@@ -72,3 +85,17 @@ def test_meta_factory_read_yaml_roundtrip(meta):
     meta2 = mf.read_pin_yaml(StringIO(pin_yaml), meta.name, meta.version)
 
     assert meta == meta2
+
+
+def test_meta_factory_roundtrip_unknown(meta):
+    meta_dict = meta.to_pin_dict()
+    meta_dict["some_other_field"] = 1
+
+    pin_yaml = yaml.dump(meta_dict)
+
+    mf = MetaFactory()
+
+    meta2 = mf.read_pin_yaml(StringIO(pin_yaml), meta.name, meta.version)
+
+    assert meta2 == meta
+    assert meta2.some_other_field == 1

setup.cfg

@@ -68,5 +68,7 @@ universal = 1
 exclude = docs
 max-line-length = 90
 ignore =
-    E501    # line too long
-    W503    # line before binary operator
+    # line too long
+    E501
+    # line before binary operator
+    W503