diff --git a/CHANGELOG.md b/CHANGELOG.md index b9cd85b1..d4d3b1a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Changed - Drop support for Python<3.6. ([#151](https://github.com/sdispater/tomlkit/pull/151)) +- Comply with TOML v1.0.0. ([#154](https://github.com/sdispater/tomlkit/pull/154)) ### Fixed diff --git a/README.md b/README.md index af7e6134..e76f3845 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ # TOML Kit - Style-preserving TOML library for Python -TOML Kit is a **1.0.0rc1-compliant** [TOML](https://github.com/toml-lang/toml) library. +TOML Kit is a **1.0.0-compliant** [TOML](https://toml.io/) library. It includes a parser that preserves all comments, indentations, whitespace and internal element ordering, and makes them accessible and editable via an intuitive API. @@ -166,6 +166,7 @@ If not, you can use `pip`: ```bash pip install tomlkit ``` + ## Running tests Please clone the repo with submodules with the following command diff --git a/poetry.lock b/poetry.lock index 1c4e162a..df6a557b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -136,6 +136,32 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "mypy" +version = "0.920" +description = "Optional static typing for Python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +mypy-extensions = ">=0.4.3,<0.5.0" +tomli = ">=1.1.0,<3.0.0" +typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} +typing-extensions = ">=3.7.4" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +python2 = ["typed-ast (>=1.4.0,<2)"] + +[[package]] +name = "mypy-extensions" +version = "0.4.3" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "nodeenv" version = "1.6.0" @@ -288,6 +314,14 @@ category = "dev" optional = false python-versions = ">=3.6" +[[package]] +name = "typed-ast" +version = "1.5.1" +description = "a fork of Python 2 and 3 ast modules with type comment support" +category = "dev" +optional = false +python-versions = ">=3.6" + [[package]] name = "typing-extensions" version = "4.0.1" @@ -332,7 +366,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "913cebf3c94a1db4435a2efa7f9ce19fc58d167a90ee9b15abacec8a9ef77e0c" +content-hash = "d2ccbad1454dffcb826fc5ddffb5efa78534b3ec2cc2e74ae8549de2ea38d0de" [metadata.files] atomicwrites = [ @@ -428,6 +462,32 @@ iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] +mypy = [ + {file = "mypy-0.920-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:41f3575b20714171c832d8f6c7aaaa0d499c9a2d1b8adaaf837b4c9065c38540"}, + {file = "mypy-0.920-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:431be889ffc8d9681813a45575c42e341c19467cbfa6dd09bf41467631feb530"}, + {file = "mypy-0.920-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f8b2059f73878e92eff7ed11a03515d6572f4338a882dd7547b5f7dd242118e6"}, + {file = "mypy-0.920-cp310-cp310-win_amd64.whl", hash = "sha256:9cd316e9705555ca6a50670ba5fb0084d756d1d8cb1697c83820b1456b0bc5f3"}, + {file = "mypy-0.920-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e091fe58b4475b3504dc7c3022ff7f4af2f9e9ddf7182047111759ed0973bbde"}, + {file = "mypy-0.920-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98b4f91a75fed2e4c6339e9047aba95968d3a7c4b91e92ab9dc62c0c583564f4"}, + {file = "mypy-0.920-cp36-cp36m-win_amd64.whl", hash = "sha256:562a0e335222d5bbf5162b554c3afe3745b495d67c7fe6f8b0d1b5bace0c1eeb"}, + {file = "mypy-0.920-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:618e677aabd21f30670bffb39a885a967337f5b112c6fb7c79375e6dced605d6"}, + {file = "mypy-0.920-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40cb062f1b7ff4cd6e897a89d8ddc48c6ad7f326b5277c93a8c559564cc1551c"}, + {file = "mypy-0.920-cp37-cp37m-win_amd64.whl", hash = "sha256:69b5a835b12fdbfeed84ef31152d41343d32ccb2b345256d8682324409164330"}, + {file = "mypy-0.920-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:993c2e52ea9570e6e872296c046c946377b9f5e89eeb7afea2a1524cf6e50b27"}, + {file = "mypy-0.920-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:df0fec878ccfcb2d1d2306ba31aa757848f681e7bbed443318d9bbd4b0d0fe9a"}, + {file = "mypy-0.920-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:331a81d2c9bf1be25317260a073b41f4584cd11701a7c14facef0aa5a005e843"}, + {file = "mypy-0.920-cp38-cp38-win_amd64.whl", hash = "sha256:ffb1e57ec49a30e3c0ebcfdc910ae4aceb7afb649310b7355509df6b15bd75f6"}, + {file = "mypy-0.920-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:31895b0b3060baf15bf76e789d94722c026f673b34b774bba9e8772295edccff"}, + {file = "mypy-0.920-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:140174e872d20d4768124a089b9f9fc83abd6a349b7f8cc6276bc344eb598922"}, + {file = "mypy-0.920-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:13b3c110309b53f5a62aa1b360f598124be33a42563b790a2a9efaacac99f1fc"}, + {file = "mypy-0.920-cp39-cp39-win_amd64.whl", hash = "sha256:82e6c15675264e923b60a11d6eb8f90665504352e68edfbb4a79aac7a04caddd"}, + {file = "mypy-0.920-py3-none-any.whl", hash = "sha256:71c77bd885d2ce44900731d4652d0d1c174dc66a0f11200e0c680bdedf1a6b37"}, + {file = "mypy-0.920.tar.gz", hash = "sha256:a55438627f5f546192f13255a994d6d1cf2659df48adcf966132b4379fd9c86b"}, +] +mypy-extensions = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] nodeenv = [ {file = "nodeenv-1.6.0-py2.py3-none-any.whl", hash = "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7"}, {file = "nodeenv-1.6.0.tar.gz", hash = "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b"}, @@ -511,6 +571,27 @@ tomli = [ {file = "tomli-1.2.3-py3-none-any.whl", hash = "sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c"}, {file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"}, ] +typed-ast = [ + {file = "typed_ast-1.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d8314c92414ce7481eee7ad42b353943679cf6f30237b5ecbf7d835519e1212"}, + {file = "typed_ast-1.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b53ae5de5500529c76225d18eeb060efbcec90ad5e030713fe8dab0fb4531631"}, + {file = "typed_ast-1.5.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:24058827d8f5d633f97223f5148a7d22628099a3d2efe06654ce872f46f07cdb"}, + {file = "typed_ast-1.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:a6d495c1ef572519a7bac9534dbf6d94c40e5b6a608ef41136133377bba4aa08"}, + {file = "typed_ast-1.5.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:de4ecae89c7d8b56169473e08f6bfd2df7f95015591f43126e4ea7865928677e"}, + {file = "typed_ast-1.5.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:256115a5bc7ea9e665c6314ed6671ee2c08ca380f9d5f130bd4d2c1f5848d695"}, + {file = "typed_ast-1.5.1-cp36-cp36m-win_amd64.whl", hash = "sha256:7c42707ab981b6cf4b73490c16e9d17fcd5227039720ca14abe415d39a173a30"}, + {file = "typed_ast-1.5.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:71dcda943a471d826ea930dd449ac7e76db7be778fcd722deb63642bab32ea3f"}, + {file = "typed_ast-1.5.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4f30a2bcd8e68adbb791ce1567fdb897357506f7ea6716f6bbdd3053ac4d9471"}, + {file = "typed_ast-1.5.1-cp37-cp37m-win_amd64.whl", hash = "sha256:ca9e8300d8ba0b66d140820cf463438c8e7b4cdc6fd710c059bfcfb1531d03fb"}, + {file = "typed_ast-1.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9caaf2b440efb39ecbc45e2fabde809cbe56272719131a6318fd9bf08b58e2cb"}, + {file = "typed_ast-1.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c9bcad65d66d594bffab8575f39420fe0ee96f66e23c4d927ebb4e24354ec1af"}, + {file = "typed_ast-1.5.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:591bc04e507595887160ed7aa8d6785867fb86c5793911be79ccede61ae96f4d"}, + {file = "typed_ast-1.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:a80d84f535642420dd17e16ae25bb46c7f4c16ee231105e7f3eb43976a89670a"}, + {file = "typed_ast-1.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:38cf5c642fa808300bae1281460d4f9b7617cf864d4e383054a5ef336e344d32"}, + {file = "typed_ast-1.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5b6ab14c56bc9c7e3c30228a0a0b54b915b1579613f6e463ba6f4eb1382e7fd4"}, + {file = "typed_ast-1.5.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a2b8d7007f6280e36fa42652df47087ac7b0a7d7f09f9468f07792ba646aac2d"}, + {file = "typed_ast-1.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:b6d17f37f6edd879141e64a5db17b67488cfeffeedad8c5cec0392305e9bc775"}, + {file = "typed_ast-1.5.1.tar.gz", hash = "sha256:484137cab8ecf47e137260daa20bafbba5f4e3ec7fda1c1e69ab299b75fa81c5"}, +] typing-extensions = [ {file = "typing_extensions-4.0.1-py3-none-any.whl", hash = "sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b"}, {file = "typing_extensions-4.0.1.tar.gz", hash = "sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e"}, diff --git a/pyproject.toml b/pyproject.toml index ebd70487..26e58e13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ pytest = "^6.2.5" pytest-cov = "^3.0.0" PyYAML = "^6.0" pre-commit = {version = "^2.1.0", python = "^3.6.1"} +mypy = "^0.920" [tool.black] line-length = 88 diff --git a/tests/conftest.py b/tests/conftest.py index 0893f59a..73333790 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,10 +43,8 @@ def _example(name): TEST_DIR = os.path.join(os.path.dirname(__file__), "toml-test", "tests") IGNORED_TESTS = { - "invalid": [ - "array-mixed-types-strings-and-ints.toml", - "array-mixed-types-arrays-and-ints.toml", - "array-mixed-types-ints-and-floats.toml", + "valid": [ + "float/inf-and-nan", # Can't compare nan ] } @@ -55,21 +53,33 @@ def get_tomltest_cases(): dirs = sorted( f for f in os.listdir(TEST_DIR) if os.path.isdir(os.path.join(TEST_DIR, f)) ) - assert dirs == ["invalid", "invalid-encoder", "valid"] - rv = {} + assert dirs == ["invalid", "valid"] + rv = {"invalid_encode": {}} for d in dirs: rv[d] = {} ignored = IGNORED_TESTS.get(d, []) - files = os.listdir(os.path.join(TEST_DIR, d)) - for f in files: - if f in ignored: - continue - - bn, ext = f.rsplit(".", 1) - if bn not in rv[d]: - rv[d][bn] = {} - with open(os.path.join(TEST_DIR, d, f), encoding="utf-8") as inp: - rv[d][bn][ext] = inp.read() + + for root, _, files in os.walk(os.path.join(TEST_DIR, d)): + relpath = os.path.relpath(root, os.path.join(TEST_DIR, d)) + if relpath == ".": + relpath = "" + for f in files: + try: + bn, ext = f.rsplit(".", 1) + except ValueError: + bn, ext = f.rsplit("-", 1) + key = f"{relpath}/{bn}" + if ext == "multi": + continue + if key in ignored: + continue + if d == "invalid" and relpath == "encoding": + rv["invalid_encode"][bn] = os.path.join(root, f) + continue + if key not in rv[d]: + rv[d][key] = {} + with open(os.path.join(root, f), encoding="utf-8") as inp: + rv[d][key][ext] = inp.read() return rv @@ -90,6 +100,6 @@ def pytest_generate_tests(metafunc): elif "invalid_encode_case" in metafunc.fixturenames: metafunc.parametrize( "invalid_encode_case", - test_list["invalid-encoder"].values(), - ids=list(test_list["invalid-encoder"].keys()), + test_list["invalid_encode"].values(), + ids=list(test_list["invalid_encode"].keys()), ) diff --git a/tests/test_items.py b/tests/test_items.py index 3cfaa34d..34e17867 100644 --- a/tests/test_items.py +++ b/tests/test_items.py @@ -17,8 +17,8 @@ from tomlkit.items import Comment from tomlkit.items import InlineTable from tomlkit.items import Integer -from tomlkit.items import Key from tomlkit.items import KeyType +from tomlkit.items import SingleKey as Key from tomlkit.items import String from tomlkit.items import StringType from tomlkit.items import Table diff --git a/tests/test_toml_tests.py b/tests/test_toml_tests.py index 12477951..88a1de35 100644 --- a/tests/test_toml_tests.py +++ b/tests/test_toml_tests.py @@ -2,6 +2,7 @@ import pytest +from tomlkit import load from tomlkit import parse from tomlkit._compat import decode from tomlkit._utils import parse_rfc3339 @@ -20,6 +21,9 @@ def to_bool(s): "integer": int, "float": float, "datetime": parse_rfc3339, + "datetime-local": parse_rfc3339, + "date-local": parse_rfc3339, + "time-local": parse_rfc3339, } @@ -50,3 +54,9 @@ def test_valid_decode(valid_case): def test_invalid_decode(invalid_decode_case): with pytest.raises(TOMLKitError): parse(invalid_decode_case["toml"]) + + +def test_invalid_encode(invalid_encode_case): + with pytest.raises((TOMLKitError, UnicodeDecodeError)): + with open(invalid_encode_case, encoding="utf-8") as f: + load(f) diff --git a/tests/toml-test b/tests/toml-test index f910e151..27837d38 160000 --- a/tests/toml-test +++ b/tests/toml-test @@ -1 +1 @@ -Subproject commit f910e151d1b14d94b1e8a4264db0814fb03520d9 +Subproject commit 27837d38aa7cc9ff385ed1316a8fef68394395dd diff --git a/tomlkit/_utils.py b/tomlkit/_utils.py index a5e1aca4..570467a1 100644 --- a/tomlkit/_utils.py +++ b/tomlkit/_utils.py @@ -15,9 +15,9 @@ "^" r"(([0-9]+)-(\d{2})-(\d{2}))?" # Date "(" - "([T ])?" # Separator + "([Tt ])?" # Separator r"(\d{2}):(\d{2}):(\d{2})(\.([0-9]+))?" # Time - r"((Z)|([\+|\-]([01][0-9]|2[0-3]):([0-5][0-9])))?" # Timezone + r"(([Zz])|([\+|\-]([01][0-9]|2[0-3]):([0-5][0-9])))?" # Timezone ")?" "$" ) @@ -25,9 +25,9 @@ RFC_3339_DATETIME = re.compile( "^" "([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])" # Date - "[T ]" # Separator + "[Tt ]" # Separator r"([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.([0-9]+))?" # Time - r"((Z)|([\+|\-]([01][0-9]|2[0-3]):([0-5][0-9])))?" # Timezone + r"(([Zz])|([\+|\-]([01][0-9]|2[0-3]):([0-5][0-9])))?" # Timezone "$" ) @@ -57,7 +57,7 @@ def parse_rfc3339(string: str) -> Union[datetime, date, time]: if m.group(9): # Timezone tz = m.group(9) - if tz == "Z": + if tz.upper() == "Z": tzinfo = _utc else: sign = m.group(11)[0] diff --git a/tomlkit/api.py b/tomlkit/api.py index d254c8e8..8a2e8f5d 100644 --- a/tomlkit/api.py +++ b/tomlkit/api.py @@ -17,6 +17,7 @@ from .items import Integer from .items import Item as _Item from .items import Key +from .items import SingleKey from .items import String from .items import Table from .items import Time @@ -141,7 +142,7 @@ def aot() -> AoT: def key(k: str) -> Key: - return Key(k) + return SingleKey(k) def value(raw: str) -> _Item: diff --git a/tomlkit/container.py b/tomlkit/container.py index 59ebddf4..dc6da897 100644 --- a/tomlkit/container.py +++ b/tomlkit/container.py @@ -18,6 +18,7 @@ from .items import Item from .items import Key from .items import Null +from .items import SingleKey from .items import Table from .items import Whitespace from .items import _CustomDict @@ -90,7 +91,7 @@ def add( def append(self, key: Union[Key, str, None], item: Item) -> "Container": if not isinstance(key, Key) and key is not None: - key = Key(key) + key = SingleKey(key) if not isinstance(item, Item): item = _item(item) @@ -255,7 +256,7 @@ def append(self, key: Union[Key, str, None], item: Item) -> "Container": def remove(self, key: Union[Key, str]) -> "Container": if not isinstance(key, Key): - key = Key(key) + key = SingleKey(key) idx = self._map.pop(key, None) if idx is None: @@ -281,10 +282,10 @@ def _insert_after( raise NonExistentKey(key) if not isinstance(key, Key): - key = Key(key) + key = SingleKey(key) if not isinstance(other_key, Key): - other_key = Key(other_key) + other_key = SingleKey(other_key) item = _item(item) @@ -323,7 +324,7 @@ def _insert_at(self, idx: int, key: Union[Key, str], item: Any) -> "Container": raise ValueError(f"Unable to insert at position {idx}") if not isinstance(key, Key): - key = Key(key) + key = SingleKey(key) item = _item(item) @@ -361,7 +362,7 @@ def _insert_at(self, idx: int, key: Union[Key, str], item: Any) -> "Container": def item(self, key: Union[Key, str]) -> Item: if not isinstance(key, Key): - key = Key(key) + key = SingleKey(key) idx = self._map.get(key, None) if idx is None: @@ -521,7 +522,7 @@ def __iter__(self) -> Iterator[str]: # Dictionary methods def __getitem__(self, key: Union[Key, str]) -> Union[Item, "Container"]: if not isinstance(key, Key): - key = Key(key) + key = SingleKey(key) idx = self._map.get(key, None) if idx is None: @@ -556,10 +557,10 @@ def _replace( self, key: Union[Key, str], new_key: Union[Key, str], value: Item ) -> None: if not isinstance(key, Key): - key = Key(key) + key = SingleKey(key) if not isinstance(new_key, Key): - new_key = Key(new_key) + new_key = SingleKey(new_key) idx = self._map.get(key, None) if idx is None: @@ -571,7 +572,7 @@ def _replace_at( self, idx: Union[int, Tuple[int]], new_key: Union[Key, str], value: Item ) -> None: if not isinstance(new_key, Key): - new_key = Key(new_key) + new_key = SingleKey(new_key) if isinstance(idx, tuple): for i in idx[1:]: diff --git a/tomlkit/items.py b/tomlkit/items.py index 4beccf0d..460feeeb 100644 --- a/tomlkit/items.py +++ b/tomlkit/items.py @@ -1,3 +1,4 @@ +import abc import re import string @@ -230,17 +231,49 @@ class KeyType(Enum): Literal = "'" -class Key: - """ - A key value. - """ +class Key(abc.ABC): + sep: str + _original: str + _keys: List["SingleKey"] + _dotted: bool + key: str + + @abc.abstractmethod + def __hash__(self) -> int: + pass + + @abc.abstractmethod + def __eq__(self, __o: object) -> bool: + pass + + def is_dotted(self) -> bool: + return self._dotted + + def __iter__(self) -> Iterator["SingleKey"]: + return iter(self._keys) + + def concat(self, other: "Key") -> "DottedKey": + keys = self._keys + other._keys + return DottedKey(keys, sep=self.sep) + + def as_string(self) -> str: + return self._original + + def __str__(self) -> str: + return self.as_string() + + def __repr__(self) -> str: + return f"" + + +class SingleKey(Key): + """A single key""" def __init__( self, k: str, t: Optional[KeyType] = None, sep: Optional[str] = None, - dotted: bool = False, original: Optional[str] = None, ) -> None: if t is None: @@ -261,36 +294,47 @@ def __init__( original = t.value + escape_quotes(k, t.value) + t.value self._original = original - - self._dotted = dotted + self._keys = [self] + self._dotted = False @property def delimiter(self) -> str: return self.t.value - def is_dotted(self) -> bool: - return self._dotted - def is_bare(self) -> bool: return self.t == KeyType.Bare - def as_string(self) -> str: - return self._original - def __hash__(self) -> int: return hash(self.key) def __eq__(self, other: Any) -> bool: if isinstance(other, Key): - return self.key == other.key + return isinstance(other, SingleKey) and self.key == other.key return self.key == other - def __str__(self) -> str: - return self.as_string() - def __repr__(self) -> str: - return f"" +class DottedKey(Key): + def __init__( + self, + keys: Iterable[Key], + sep: Optional[str] = None, + original: Optional[str] = None, + ) -> None: + self._keys = list(keys) + if original is None: + original = ".".join(k.as_string() for k in self._keys) + + self.sep = "=" if sep is None else sep + self._original = original + self._dotted = True + self.key = ".".join(k.key for k in self._keys) + + def __hash__(self) -> int: + return hash(tuple(self._keys)) + + def __eq__(self, __o: object) -> bool: + return isinstance(__o, DottedKey) and self._keys == __o._keys class Item: @@ -1426,7 +1470,8 @@ def __delitem__(self, key: Union[slice, int]) -> None: del self._body[key] list.__delitem__(self, key) - def insert(self, index: int, value: Table) -> None: + def insert(self, index: int, value: dict) -> None: + value = item(value, _parent=self) if not isinstance(value, Table): raise ValueError(f"Unsupported insert value type: {type(value)}") length = len(self) diff --git a/tomlkit/parser.py b/tomlkit/parser.py index ed6e6213..da1e5c47 100644 --- a/tomlkit/parser.py +++ b/tomlkit/parser.py @@ -2,7 +2,6 @@ import string from typing import Any -from typing import Iterator from typing import List from typing import Optional from typing import Tuple @@ -14,6 +13,7 @@ from ._utils import _escaped from ._utils import parse_rfc3339 from .container import Container +from .exceptions import EmptyKeyError from .exceptions import EmptyTableNameError from .exceptions import InternalParserError from .exceptions import InvalidCharInStringError @@ -40,6 +40,7 @@ from .items import Key from .items import KeyType from .items import Null +from .items import SingleKey from .items import String from .items import StringType from .items import Table @@ -67,7 +68,7 @@ def __init__(self, string: str) -> None: # Input to parse self._src = Source(decode(string)) - self._aot_stack = [] + self._aot_stack: List[Key] = [] @property def _state(self): @@ -157,7 +158,7 @@ def parse(self) -> TOMLDocument: if isinstance(value, Table) and value.is_aot_element(): # This is just the first table in an AoT. Parse the rest of the array # along with it. - value = self._parse_aot(value, key.key) + value = self._parse_aot(value, key) body.append(key, value) @@ -187,76 +188,19 @@ def _merge_ws(self, item: Item, container: Container) -> bool: return True - def _is_child(self, parent: str, child: str) -> bool: + def _is_child(self, parent: Key, child: Key) -> bool: """ Returns whether a key is strictly a child of another key. AoT siblings are not considered children of one another. """ - parent_parts = tuple(self._split_table_name(parent)) - child_parts = tuple(self._split_table_name(child)) + parent_parts = tuple(parent) + child_parts = tuple(child) if parent_parts == child_parts: return False return parent_parts == child_parts[: len(parent_parts)] - def _split_table_name(self, name: str) -> Iterator[Key]: - in_name = False - current = "" - original = "" - t = KeyType.Bare - parts = 0 - for c in name: - c = TOMLChar(c) - - if c == ".": - if in_name: - current += c - original += c - continue - - if not current: - raise self.parse_error() - - yield Key(current.strip(), t=t, sep="", original=original) - - parts += 1 - - current = original = "" - t = KeyType.Bare - elif c in {"'", '"'}: - if in_name: - if c == t.value: - in_name = False - else: - current += c - else: - if ( - current.strip() - and TOMLChar(current[-1]).is_spaces() - and not parts - ): - raise self.parse_error() - - in_name = True - t = KeyType.Literal if c == "'" else KeyType.Basic - original += c - elif in_name or c.is_bare_key_char(): - current += c - original += c - elif c.is_spaces(): - # A space is only valid at this point - # if it's in between parts. - # We store it for now and will check - # later if it's valid - current += c - original += c - else: - raise self.parse_error() - - if current.strip(): - yield Key(current.strip(), t=t, sep="", original=original) - def _parse_item(self) -> Optional[Tuple[Optional[Key], Item]]: """ Attempts to parse the next item and returns it, along with its key @@ -406,6 +350,10 @@ def _parse_key(self) -> Key: Parses a Key at the current position; WS before the key must be exhausted first at the callsite. """ + self.mark() + while self._current.is_spaces() and self.inc(): + # Skip any leading whitespace + pass if self._current in "\"'": return self._parse_quoted_key() else: @@ -415,43 +363,35 @@ def _parse_quoted_key(self) -> Key: """ Parses a key enclosed in either single or double quotes. """ + # Extract the leading whitespace + original = self.extract() quote_style = self._current - key_type = None - dotted = False - for t in KeyType: - if t.value == quote_style: - key_type = t - break + key_type = next((t for t in KeyType if t.value == quote_style), None) if key_type is None: raise RuntimeError("Should not have entered _parse_quoted_key()") - self.inc() + key_str = self._parse_string( + StringType.SLB if key_type == KeyType.Basic else StringType.SLL + ) + if key_str._t.is_multiline(): + raise self.parse_error(UnexpectedCharError, key_str._t.value) + original += key_str.as_string() self.mark() - - while self._current != quote_style and self.inc(): + while self._current.is_spaces() and self.inc(): pass - - key = self.extract() - + original += self.extract() + key = SingleKey(str(key_str), t=key_type, sep="", original=original) if self._current == ".": self.inc() - dotted = True - key += "." + self._parse_key().as_string() - key_type = KeyType.Bare - else: - self.inc() + key = key.concat(self._parse_key()) - return Key(key, key_type, "", dotted) + return key def _parse_bare_key(self) -> Key: """ Parses a bare key. """ - key_type = None - dotted = False - - self.mark() while ( self._current.is_bare_key_char() or self._current.is_spaces() ) and self.inc(): @@ -461,25 +401,24 @@ def _parse_bare_key(self) -> Key: key = original.strip() if not key: # Empty key - raise self.parse_error(ParseError, "Empty key found") + raise self.parse_error(EmptyKeyError) if " " in key: # Bare key with spaces in it raise self.parse_error(ParseError, f'Invalid key "{key}"') + key = SingleKey(key, KeyType.Bare, "", original) + if self._current == ".": self.inc() - dotted = True - original += "." + self._parse_key().as_string() - key = original.strip() - key_type = KeyType.Bare + key = key.concat(self._parse_key()) - return Key(key, key_type, "", dotted, original=original) + return key def _handle_dotted_key( self, container: Union[Container, Table], key: Key, value: Any ) -> None: - names = tuple(self._split_table_name(key.as_string())) + names = tuple(iter(key)) name = names[0] name._dotted = True if name in container: @@ -796,10 +735,11 @@ def _parse_number(self, raw: str, trivia: Trivia) -> Optional[Item]: sign = raw[0] raw = raw[1:] - if ( - len(raw) > 1 - and raw.startswith("0") + if len(raw) > 1 and ( + raw.startswith("0") and not raw.startswith(("0.", "0o", "0x", "0b", "0e")) + or sign + and raw.startswith(".") ): return @@ -819,12 +759,16 @@ def _parse_number(self, raw: str, trivia: Trivia) -> Optional[Item]: base = 16 # Underscores should be surrounded by digits - clean = re.sub(f"(?i)(?<={digits})_(?={digits})", "", raw) + clean = re.sub(f"(?i)(?<={digits})_(?={digits})", "", raw).lower() if "_" in clean: return - if clean.endswith("."): + if ( + clean.endswith(".") + or not clean.startswith("0x") + and clean.split("e", 1)[0].endswith(".") + ): return try: @@ -996,7 +940,7 @@ def _parse_string(self, delim: StringType) -> String: self.inc(exception=UnexpectedEofError) def _parse_table( - self, parent_name: Optional[str] = None, parent: Optional[Table] = None + self, parent_name: Optional[Key] = None, parent: Optional[Table] = None ) -> Tuple[Key, Union[Table, AoT]]: """ Parses a table element. @@ -1018,58 +962,28 @@ def _parse_table( raise self.parse_error(UnexpectedEofError) is_aot = True - - # Consume any whitespace - self.mark() - while self._current.is_spaces() and self.inc(): - pass - - ws_prefix = self.extract() - - # Key - if self._current in [StringType.SLL.value, StringType.SLB.value]: - delimiter = ( - StringType.SLL - if self._current == StringType.SLL.value - else StringType.SLB - ) - name = self._parse_string(delimiter) - name = "{delimiter}{name}{delimiter}".format( - delimiter=delimiter.value, name=name - ) - - self.mark() - while self._current != "]" and self.inc(): - if self.end(): - raise self.parse_error(UnexpectedEofError) - - pass - - ws_suffix = self.extract() - name += ws_suffix - else: - self.mark() - while self._current != "]" and self.inc(): - if self.end(): - raise self.parse_error(UnexpectedEofError) - - pass - - name = self.extract() - - name = ws_prefix + name - - if not name.strip(): + try: + key = self._parse_key() + except EmptyKeyError: + raise self.parse_error(EmptyTableNameError) from None + if self.end(): + raise self.parse_error(UnexpectedEofError) + elif self._current != "]": + raise self.parse_error(UnexpectedCharError, self._current) + elif not key.key.strip(): raise self.parse_error(EmptyTableNameError) - key = Key(name, sep="") - name_parts = tuple(self._split_table_name(name)) + key.sep = "" + full_key = key + name_parts = tuple(key) if any(" " in part.key.strip() and part.is_bare() for part in name_parts): - raise self.parse_error(ParseError, f'Invalid table name "{name}"') + raise self.parse_error( + ParseError, f'Invalid table name "{full_key.as_string()}"' + ) missing_table = False if parent_name: - parent_name_parts = tuple(self._split_table_name(parent_name)) + parent_name_parts = tuple(parent_name) else: parent_name_parts = tuple() @@ -1093,7 +1007,7 @@ def _parse_table( Trivia(indent, cws, comment, trail), is_aot, name=name_parts[0].key if name_parts else key.key, - display_name=name, + display_name=full_key.as_string(), ) if len(name_parts) > 1: @@ -1106,36 +1020,36 @@ def _parse_table( table = Table( Container(True), Trivia(indent, cws, comment, trail), - is_aot and name_parts[0].key in self._aot_stack, + is_aot and name_parts[0] in self._aot_stack, is_super_table=True, name=name_parts[0].key, ) - result = table - key = name_parts[0] + result = table + key = name_parts[0] - for i, _name in enumerate(name_parts[1:]): - if _name in table: - child = table[_name] - else: - child = Table( - Container(True), - Trivia(indent, cws, comment, trail), - is_aot and i == len(name_parts) - 2, - is_super_table=i < len(name_parts) - 2, - name=_name.key, - display_name=name if i == len(name_parts) - 2 else None, - ) + for i, _name in enumerate(name_parts[1:]): + if _name in table: + child = table[_name] + else: + child = Table( + Container(True), + Trivia(indent, cws, comment, trail), + is_aot and i == len(name_parts) - 2, + is_super_table=i < len(name_parts) - 2, + name=_name.key, + display_name=full_key.as_string() + if i == len(name_parts) - 2 + else None, + ) - if is_aot and i == len(name_parts) - 2: - table.raw_append( - _name, AoT([child], name=table.name, parsed=True) - ) - else: - table.raw_append(_name, child) + if is_aot and i == len(name_parts) - 2: + table.raw_append(_name, AoT([child], name=table.name, parsed=True)) + else: + table.raw_append(_name, child) - table = child - values = table.value + table = child + values = table.value else: if name_parts: key = name_parts[0] @@ -1151,21 +1065,21 @@ def _parse_table( table.raw_append(_key, item) else: if self._current == "[": - is_aot_next, name_next = self._peek_table() + _, key_next = self._peek_table() - if self._is_child(name, name_next): - key_next, table_next = self._parse_table(name, table) + if self._is_child(full_key, key_next): + key_next, table_next = self._parse_table(full_key, table) table.raw_append(key_next, table_next) # Picking up any sibling while not self.end(): - _, name_next = self._peek_table() + _, key_next = self._peek_table() - if not self._is_child(name, name_next): + if not self._is_child(full_key, key_next): break - key_next, table_next = self._parse_table(name, table) + key_next, table_next = self._parse_table(full_key, table) table.raw_append(key_next, table_next) @@ -1179,12 +1093,12 @@ def _parse_table( if isinstance(result, Null): result = table - if is_aot and (not self._aot_stack or name != self._aot_stack[-1]): - result = self._parse_aot(result, name) + if is_aot and (not self._aot_stack or full_key != self._aot_stack[-1]): + result = self._parse_aot(result, full_key) return key, result - def _peek_table(self) -> Tuple[bool, str]: + def _peek_table(self) -> Tuple[bool, Key]: """ Peeks ahead non-intrusively by cloning then restoring the initial state of the parser. @@ -1206,16 +1120,12 @@ def _peek_table(self) -> Tuple[bool, str]: if self._current == "[": self.inc() is_aot = True + try: + return is_aot, self._parse_key() + except EmptyKeyError: + raise self.parse_error(EmptyTableNameError) from None - self.mark() - - table_name = "" - while self._current != "]" and self.inc(): - table_name = self.extract() - - return is_aot, table_name - - def _parse_aot(self, first: Table, name_first: str) -> AoT: + def _parse_aot(self, first: Table, name_first: Key) -> AoT: """ Parses all siblings of the provided table first and bundles them into an AoT. diff --git a/tox.ini b/tox.ini index 0b50f48a..7a653b43 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] skipsdist = True -envlist = py27, py34, py35, py36, py37, py38, pypy3 +envlist = py36, py37, py38, py39, pypy3 [testenv] whitelist_externals = poetry