diff --git a/poetry.lock b/poetry.lock index 5dad14af..dcb81531 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,16 @@ # This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +[[package]] +name = "appnope" +version = "0.1.3" +description = "Disable App Nap on macOS >= 10.9" +optional = false +python-versions = "*" +files = [ + {file = "appnope-0.1.3-py2.py3-none-any.whl", hash = "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e"}, + {file = "appnope-0.1.3.tar.gz", hash = "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24"}, +] + [[package]] name = "argcomplete" version = "3.1.1" @@ -33,11 +44,28 @@ wrapt = [ {version = ">=1.14,<2", markers = "python_version >= \"3.11\""}, ] +[[package]] +name = "asttokens" +version = "2.2.1" +description = "Annotate AST trees with source code positions" +optional = false +python-versions = "*" +files = [ + {file = "asttokens-2.2.1-py2.py3-none-any.whl", hash = "sha256:6b0ac9e93fb0335014d382b8fa9b3afa7df546984258005da0b9e7095b3deb1c"}, + {file = "asttokens-2.2.1.tar.gz", hash = "sha256:4622110b2a6f30b77e1473affaa97e711bc2f07d3f10848420ff1898edbe94f3"}, +] + +[package.dependencies] +six = "*" + +[package.extras] +test = ["astroid", "pytest"] + [[package]] name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, @@ -51,6 +79,17 @@ docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib- tests = ["attrs[tests-no-zope]", "zope-interface"] tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +[[package]] +name = "backcall" +version = "0.2.0" +description = "Specifications for callback functions passed in to an API" +optional = false +python-versions = "*" +files = [ + {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, + {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"}, +] + [[package]] name = "bandit" version = "1.7.5" @@ -74,6 +113,42 @@ test = ["beautifulsoup4 (>=4.8.0)", "coverage (>=4.5.4)", "fixtures (>=3.0.0)", toml = ["tomli (>=1.1.0)"] yaml = ["PyYAML"] +[[package]] +name = "beautifulsoup4" +version = "4.12.2" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.2-py3-none-any.whl", hash = "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"}, + {file = "beautifulsoup4-4.12.2.tar.gz", hash = "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + +[[package]] +name = "bleach" +version = "6.0.0" +description = "An easy safelist-based HTML-sanitizing tool." +optional = false +python-versions = ">=3.7" +files = [ + {file = "bleach-6.0.0-py3-none-any.whl", hash = "sha256:33c16e3353dbd13028ab4799a0f89a83f113405c766e9c122df8a06f5b85b3f4"}, + {file = "bleach-6.0.0.tar.gz", hash = "sha256:1a1a85c1595e07d8db14c5f09f09e6433502c51c595970edc090551f0db99414"}, +] + +[package.dependencies] +six = ">=1.9.0" +webencodings = "*" + +[package.extras] +css = ["tinycss2 (>=1.1.0,<1.2)"] + [[package]] name = "brotli" version = "1.0.9" @@ -273,7 +348,7 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." -optional = true +optional = false python-versions = "*" files = [ {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, @@ -440,6 +515,25 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "comm" +version = "0.1.4" +description = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc." +optional = false +python-versions = ">=3.6" +files = [ + {file = "comm-0.1.4-py3-none-any.whl", hash = "sha256:6d52794cba11b36ed9860999cd10fd02d6b2eac177068fdd585e1e2f8a96e67a"}, + {file = "comm-0.1.4.tar.gz", hash = "sha256:354e40a59c9dd6db50c5cc6b4acc887d82e9603787f83b68c01a80a923984d15"}, +] + +[package.dependencies] +traitlets = ">=4" + +[package.extras] +lint = ["black (>=22.6.0)", "mdformat (>0.7)", "mdformat-gfm (>=0.3.5)", "ruff (>=0.0.156)"] +test = ["pytest"] +typing = ["mypy (>=0.990)"] + [[package]] name = "contourpy" version = "1.1.0" @@ -528,6 +622,44 @@ files = [ {file = "cycler-0.11.0.tar.gz", hash = "sha256:9c87405839a19696e837b3b818fed3f5f69f16f1eec1a1ad77e043dcea9c772f"}, ] +[[package]] +name = "debugpy" +version = "1.6.7.post1" +description = "An implementation of the Debug Adapter Protocol for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "debugpy-1.6.7.post1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:903bd61d5eb433b6c25b48eae5e23821d4c1a19e25c9610205f5aeaccae64e32"}, + {file = "debugpy-1.6.7.post1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d16882030860081e7dd5aa619f30dec3c2f9a421e69861125f83cc372c94e57d"}, + {file = "debugpy-1.6.7.post1-cp310-cp310-win32.whl", hash = "sha256:eea8d8cfb9965ac41b99a61f8e755a8f50e9a20330938ad8271530210f54e09c"}, + {file = "debugpy-1.6.7.post1-cp310-cp310-win_amd64.whl", hash = "sha256:85969d864c45f70c3996067cfa76a319bae749b04171f2cdeceebe4add316155"}, + {file = "debugpy-1.6.7.post1-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:890f7ab9a683886a0f185786ffbda3b46495c4b929dab083b8c79d6825832a52"}, + {file = "debugpy-1.6.7.post1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4ac7a4dba28801d184b7fc0e024da2635ca87d8b0a825c6087bb5168e3c0d28"}, + {file = "debugpy-1.6.7.post1-cp37-cp37m-win32.whl", hash = "sha256:3370ef1b9951d15799ef7af41f8174194f3482ee689988379763ef61a5456426"}, + {file = "debugpy-1.6.7.post1-cp37-cp37m-win_amd64.whl", hash = "sha256:65b28435a17cba4c09e739621173ff90c515f7b9e8ea469b92e3c28ef8e5cdfb"}, + {file = "debugpy-1.6.7.post1-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:92b6dae8bfbd497c90596bbb69089acf7954164aea3228a99d7e43e5267f5b36"}, + {file = "debugpy-1.6.7.post1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72f5d2ecead8125cf669e62784ef1e6300f4067b0f14d9f95ee00ae06fc7c4f7"}, + {file = "debugpy-1.6.7.post1-cp38-cp38-win32.whl", hash = "sha256:f0851403030f3975d6e2eaa4abf73232ab90b98f041e3c09ba33be2beda43fcf"}, + {file = "debugpy-1.6.7.post1-cp38-cp38-win_amd64.whl", hash = "sha256:3de5d0f97c425dc49bce4293df6a04494309eedadd2b52c22e58d95107e178d9"}, + {file = "debugpy-1.6.7.post1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:38651c3639a4e8bbf0ca7e52d799f6abd07d622a193c406be375da4d510d968d"}, + {file = "debugpy-1.6.7.post1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:038c51268367c9c935905a90b1c2d2dbfe304037c27ba9d19fe7409f8cdc710c"}, + {file = "debugpy-1.6.7.post1-cp39-cp39-win32.whl", hash = "sha256:4b9eba71c290852f959d2cf8a03af28afd3ca639ad374d393d53d367f7f685b2"}, + {file = "debugpy-1.6.7.post1-cp39-cp39-win_amd64.whl", hash = "sha256:973a97ed3b434eab0f792719a484566c35328196540676685c975651266fccf9"}, + {file = "debugpy-1.6.7.post1-py2.py3-none-any.whl", hash = "sha256:1093a5c541af079c13ac8c70ab8b24d1d35c8cacb676306cf11e57f699c02926"}, + {file = "debugpy-1.6.7.post1.zip", hash = "sha256:fe87ec0182ef624855d05e6ed7e0b7cb1359d2ffa2a925f8ec2d22e98b75d0ca"}, +] + +[[package]] +name = "decorator" +version = "5.1.1" +description = "Decorators for Humans" +optional = false +python-versions = ">=3.5" +files = [ + {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, + {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, +] + [[package]] name = "deepmerge" version = "1.1.0" @@ -539,6 +671,17 @@ files = [ {file = "deepmerge-1.1.0.tar.gz", hash = "sha256:4c27a0db5de285e1a7ceac7dbc1531deaa556b627dea4900c8244581ecdfea2d"}, ] +[[package]] +name = "defusedxml" +version = "0.7.1" +description = "XML bomb protection for Python stdlib modules" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, + {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, +] + [[package]] name = "deprecation" version = "2.1.0" @@ -621,11 +764,25 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "executing" +version = "1.2.0" +description = "Get the currently executing AST node of a frame, and other information" +optional = false +python-versions = "*" +files = [ + {file = "executing-1.2.0-py2.py3-none-any.whl", hash = "sha256:0314a69e37426e3608aada02473b4161d4caf5a4b244d1d0c48072b8fee7bacc"}, + {file = "executing-1.2.0.tar.gz", hash = "sha256:19da64c18d2d851112f09c287f8d3dbbdf725ab0e569077efb6cdcbd3497c107"}, +] + +[package.extras] +tests = ["asttokens", "littleutils", "pytest", "rich"] + [[package]] name = "fastjsonschema" version = "2.18.0" description = "Fastest Python implementation of JSON schema" -optional = true +optional = false python-versions = "*" files = [ {file = "fastjsonschema-2.18.0-py3-none-any.whl", hash = "sha256:128039912a11a807068a7c87d0da36660afbfd7202780db26c4aa7153cfdc799"}, @@ -844,6 +1001,25 @@ pyav = ["av"] test = ["fsspec[github]", "pytest", "pytest-cov"] tifffile = ["tifffile"] +[[package]] +name = "importlib-metadata" +version = "6.8.0" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "importlib_metadata-6.8.0-py3-none-any.whl", hash = "sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb"}, + {file = "importlib_metadata-6.8.0.tar.gz", hash = "sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743"}, +] + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] + [[package]] name = "importlib-resources" version = "6.0.0" @@ -873,6 +1049,78 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "ipykernel" +version = "6.25.1" +description = "IPython Kernel for Jupyter" +optional = false +python-versions = ">=3.8" +files = [ + {file = "ipykernel-6.25.1-py3-none-any.whl", hash = "sha256:c8a2430b357073b37c76c21c52184db42f6b4b0e438e1eb7df3c4440d120497c"}, + {file = "ipykernel-6.25.1.tar.gz", hash = "sha256:050391364c0977e768e354bdb60cbbfbee7cbb943b1af1618382021136ffd42f"}, +] + +[package.dependencies] +appnope = {version = "*", markers = "platform_system == \"Darwin\""} +comm = ">=0.1.1" +debugpy = ">=1.6.5" +ipython = ">=7.23.1" +jupyter-client = ">=6.1.12" +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +matplotlib-inline = ">=0.1" +nest-asyncio = "*" +packaging = "*" +psutil = "*" +pyzmq = ">=20" +tornado = ">=6.1" +traitlets = ">=5.4.0" + +[package.extras] +cov = ["coverage[toml]", "curio", "matplotlib", "pytest-cov", "trio"] +docs = ["myst-parser", "pydata-sphinx-theme", "sphinx", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "trio"] +pyqt5 = ["pyqt5"] +pyside6 = ["pyside6"] +test = ["flaky", "ipyparallel", "pre-commit", "pytest (>=7.0)", "pytest-asyncio", "pytest-cov", "pytest-timeout"] + +[[package]] +name = "ipython" +version = "8.14.0" +description = "IPython: Productive Interactive Computing" +optional = false +python-versions = ">=3.9" +files = [ + {file = "ipython-8.14.0-py3-none-any.whl", hash = "sha256:248aca623f5c99a6635bc3857677b7320b9b8039f99f070ee0d20a5ca5a8e6bf"}, + {file = "ipython-8.14.0.tar.gz", hash = "sha256:1d197b907b6ba441b692c48cf2a3a2de280dc0ac91a3405b39349a50272ca0a1"}, +] + +[package.dependencies] +appnope = {version = "*", markers = "sys_platform == \"darwin\""} +backcall = "*" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +decorator = "*" +jedi = ">=0.16" +matplotlib-inline = "*" +pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} +pickleshare = "*" +prompt-toolkit = ">=3.0.30,<3.0.37 || >3.0.37,<3.1.0" +pygments = ">=2.4.0" +stack-data = "*" +traitlets = ">=5" +typing-extensions = {version = "*", markers = "python_version < \"3.10\""} + +[package.extras] +all = ["black", "curio", "docrepr", "ipykernel", "ipyparallel", "ipywidgets", "matplotlib", "matplotlib (!=3.2.0)", "nbconvert", "nbformat", "notebook", "numpy (>=1.21)", "pandas", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio", "qtconsole", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "trio", "typing-extensions"] +black = ["black"] +doc = ["docrepr", "ipykernel", "matplotlib", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "typing-extensions"] +kernel = ["ipykernel"] +nbconvert = ["nbconvert"] +nbformat = ["nbformat"] +notebook = ["ipywidgets", "notebook"] +parallel = ["ipyparallel"] +qtconsole = ["qtconsole"] +test = ["pytest (<7.1)", "pytest-asyncio", "testpath"] +test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.21)", "pandas", "pytest (<7.1)", "pytest-asyncio", "testpath", "trio"] + [[package]] name = "isort" version = "5.12.0" @@ -890,11 +1138,30 @@ pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib" plugins = ["setuptools"] requirements-deprecated-finder = ["pip-api", "pipreqs"] +[[package]] +name = "jedi" +version = "0.19.0" +description = "An autocompletion tool for Python that can be used for text editors." +optional = false +python-versions = ">=3.6" +files = [ + {file = "jedi-0.19.0-py2.py3-none-any.whl", hash = "sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e"}, + {file = "jedi-0.19.0.tar.gz", hash = "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4"}, +] + +[package.dependencies] +parso = ">=0.8.3,<0.9.0" + +[package.extras] +docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx (==1.8.5)", "sphinx-rtd-theme (==0.4.3)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"] +qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] +testing = ["Django (<3.1)", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] + [[package]] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, @@ -911,7 +1178,7 @@ i18n = ["Babel (>=2.7)"] name = "jsonschema" version = "4.18.4" description = "An implementation of JSON Schema validation for Python" -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "jsonschema-4.18.4-py3-none-any.whl", hash = "sha256:971be834317c22daaa9132340a51c01b50910724082c2c1a2ac87eeec153a3fe"}, @@ -932,7 +1199,7 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "jsonschema-specifications" version = "2023.7.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "jsonschema_specifications-2023.7.1-py3-none-any.whl", hash = "sha256:05adf340b659828a004220a9613be00fa3f223f2b82002e273dee62fd50524b1"}, @@ -942,11 +1209,34 @@ files = [ [package.dependencies] referencing = ">=0.28.0" +[[package]] +name = "jupyter-client" +version = "8.3.0" +description = "Jupyter protocol implementation and client libraries" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jupyter_client-8.3.0-py3-none-any.whl", hash = "sha256:7441af0c0672edc5d28035e92ba5e32fadcfa8a4e608a434c228836a89df6158"}, + {file = "jupyter_client-8.3.0.tar.gz", hash = "sha256:3af69921fe99617be1670399a0b857ad67275eefcfa291e2c81a160b7b650f5f"}, +] + +[package.dependencies] +importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.10\""} +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +python-dateutil = ">=2.8.2" +pyzmq = ">=23.0" +tornado = ">=6.2" +traitlets = ">=5.3" + +[package.extras] +docs = ["ipykernel", "myst-parser", "pydata-sphinx-theme", "sphinx (>=4)", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] +test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"] + [[package]] name = "jupyter-core" version = "5.3.1" description = "Jupyter core package. A base package on which Jupyter projects rely." -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "jupyter_core-5.3.1-py3-none-any.whl", hash = "sha256:ae9036db959a71ec1cac33081eeb040a79e681f08ab68b0883e9a676c7a90dce"}, @@ -962,6 +1252,17 @@ traitlets = ">=5.3" docs = ["myst-parser", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "traitlets"] test = ["ipykernel", "pre-commit", "pytest", "pytest-cov", "pytest-timeout"] +[[package]] +name = "jupyterlab-pygments" +version = "0.2.2" +description = "Pygments theme using JupyterLab CSS variables" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jupyterlab_pygments-0.2.2-py2.py3-none-any.whl", hash = "sha256:2405800db07c9f770863bcf8049a529c3dd4d3e28536638bd7c1c01d2748309f"}, + {file = "jupyterlab_pygments-0.2.2.tar.gz", hash = "sha256:7405d7fde60819d905a9fa8ce89e4cd830e318cdad22a0030f7a901da705585d"}, +] + [[package]] name = "kiwisolver" version = "1.4.4" @@ -1370,6 +1671,20 @@ pillow = ">=6.2.0" pyparsing = ">=2.3.1,<3.1" python-dateutil = ">=2.7" +[[package]] +name = "matplotlib-inline" +version = "0.1.6" +description = "Inline Matplotlib backend for Jupyter" +optional = false +python-versions = ">=3.5" +files = [ + {file = "matplotlib-inline-0.1.6.tar.gz", hash = "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304"}, + {file = "matplotlib_inline-0.1.6-py3-none-any.whl", hash = "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311"}, +] + +[package.dependencies] +traitlets = "*" + [[package]] name = "mccabe" version = "0.7.0" @@ -1392,6 +1707,17 @@ files = [ {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, ] +[[package]] +name = "mistune" +version = "3.0.1" +description = "A sane and fast Markdown parser with useful plugins and renderers" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mistune-3.0.1-py3-none-any.whl", hash = "sha256:b9b3e438efbb57c62b5beb5e134dab664800bdf1284a7ee09e8b12b13eb1aac6"}, + {file = "mistune-3.0.1.tar.gz", hash = "sha256:e912116c13aa0944f9dc530db38eb88f6a77087ab128f49f84a48f4c05ea163c"}, +] + [[package]] name = "mypy" version = "1.4.1" @@ -1464,11 +1790,71 @@ files = [ fast = ["fastnumbers (>=2.0.0)"] icu = ["PyICU (>=1.0.0)"] +[[package]] +name = "nbclient" +version = "0.8.0" +description = "A client library for executing notebooks. Formerly nbconvert's ExecutePreprocessor." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "nbclient-0.8.0-py3-none-any.whl", hash = "sha256:25e861299e5303a0477568557c4045eccc7a34c17fc08e7959558707b9ebe548"}, + {file = "nbclient-0.8.0.tar.gz", hash = "sha256:f9b179cd4b2d7bca965f900a2ebf0db4a12ebff2f36a711cb66861e4ae158e55"}, +] + +[package.dependencies] +jupyter-client = ">=6.1.12" +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +nbformat = ">=5.1" +traitlets = ">=5.4" + +[package.extras] +dev = ["pre-commit"] +docs = ["autodoc-traits", "mock", "moto", "myst-parser", "nbclient[test]", "sphinx (>=1.7)", "sphinx-book-theme", "sphinxcontrib-spelling"] +test = ["flaky", "ipykernel (>=6.19.3)", "ipython", "ipywidgets", "nbconvert (>=7.0.0)", "pytest (>=7.0)", "pytest-asyncio", "pytest-cov (>=4.0)", "testpath", "xmltodict"] + +[[package]] +name = "nbconvert" +version = "7.7.3" +description = "Converting Jupyter Notebooks" +optional = false +python-versions = ">=3.8" +files = [ + {file = "nbconvert-7.7.3-py3-none-any.whl", hash = "sha256:3022adadff3f86578a47fab7c2228bb3ca9c56a24345642a22f917f6168b48fc"}, + {file = "nbconvert-7.7.3.tar.gz", hash = "sha256:4a5996bf5f3cd16aa0431897ba1aa4c64842c2079f434b3dc6b8c4b252ef3355"}, +] + +[package.dependencies] +beautifulsoup4 = "*" +bleach = "!=5.0.0" +defusedxml = "*" +importlib-metadata = {version = ">=3.6", markers = "python_version < \"3.10\""} +jinja2 = ">=3.0" +jupyter-core = ">=4.7" +jupyterlab-pygments = "*" +markupsafe = ">=2.0" +mistune = ">=2.0.3,<4" +nbclient = ">=0.5.0" +nbformat = ">=5.7" +packaging = "*" +pandocfilters = ">=1.4.1" +pygments = ">=2.4.1" +tinycss2 = "*" +traitlets = ">=5.1" + +[package.extras] +all = ["nbconvert[docs,qtpdf,serve,test,webpdf]"] +docs = ["ipykernel", "ipython", "myst-parser", "nbsphinx (>=0.2.12)", "pydata-sphinx-theme", "sphinx (==5.0.2)", "sphinxcontrib-spelling"] +qtpdf = ["nbconvert[qtpng]"] +qtpng = ["pyqtwebengine (>=5.15)"] +serve = ["tornado (>=6.1)"] +test = ["flaky", "ipykernel", "ipywidgets (>=7)", "pre-commit", "pytest", "pytest-dependency"] +webpdf = ["playwright"] + [[package]] name = "nbformat" version = "5.9.2" description = "The Jupyter Notebook format" -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "nbformat-5.9.2-py3-none-any.whl", hash = "sha256:1c5172d786a41b82bcfd0c23f9e6b6f072e8fb49c39250219e4acfff1efe89e9"}, @@ -1485,6 +1871,17 @@ traitlets = ">=5.1" docs = ["myst-parser", "pydata-sphinx-theme", "sphinx", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] test = ["pep440", "pre-commit", "pytest", "testpath"] +[[package]] +name = "nest-asyncio" +version = "1.5.7" +description = "Patch asyncio to allow nested event loops" +optional = false +python-versions = ">=3.5" +files = [ + {file = "nest_asyncio-1.5.7-py3-none-any.whl", hash = "sha256:5301c82941b550b3123a1ea772ba9a1c80bad3a182be8c1a5ae6ad3be57a9657"}, + {file = "nest_asyncio-1.5.7.tar.gz", hash = "sha256:6a80f7b98f24d9083ed24608977c09dd608d83f91cccc24c9d2cba6d10e01c10"}, +] + [[package]] name = "networkx" version = "3.1" @@ -1575,6 +1972,32 @@ files = [ {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] +[[package]] +name = "pandocfilters" +version = "1.5.0" +description = "Utilities for writing pandoc filters in python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pandocfilters-1.5.0-py2.py3-none-any.whl", hash = "sha256:33aae3f25fd1a026079f5d27bdd52496f0e0803b3469282162bafdcbdf6ef14f"}, + {file = "pandocfilters-1.5.0.tar.gz", hash = "sha256:0b679503337d233b4339a817bfc8c50064e2eff681314376a47cb582305a7a38"}, +] + +[[package]] +name = "parso" +version = "0.8.3" +description = "A Python Parser" +optional = false +python-versions = ">=3.6" +files = [ + {file = "parso-0.8.3-py2.py3-none-any.whl", hash = "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"}, + {file = "parso-0.8.3.tar.gz", hash = "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0"}, +] + +[package.extras] +qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] +testing = ["docopt", "pytest (<6.0.0)"] + [[package]] name = "pastedeploy" version = "3.0.1" @@ -1627,6 +2050,31 @@ files = [ [package.dependencies] flake8-polyfill = ">=1.0.2,<2" +[[package]] +name = "pexpect" +version = "4.8.0" +description = "Pexpect allows easy control of interactive console applications." +optional = false +python-versions = "*" +files = [ + {file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"}, + {file = "pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"}, +] + +[package.dependencies] +ptyprocess = ">=0.5" + +[[package]] +name = "pickleshare" +version = "0.7.5" +description = "Tiny 'shelve'-like database with concurrency support" +optional = false +python-versions = "*" +files = [ + {file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"}, + {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, +] + [[package]] name = "pikepdf" version = "8.2.3" @@ -1820,6 +2268,20 @@ files = [ [package.extras] twisted = ["twisted"] +[[package]] +name = "prompt-toolkit" +version = "3.0.39" +description = "Library for building powerful interactive command lines in Python" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "prompt_toolkit-3.0.39-py3-none-any.whl", hash = "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"}, + {file = "prompt_toolkit-3.0.39.tar.gz", hash = "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac"}, +] + +[package.dependencies] +wcwidth = "*" + [[package]] name = "prospector" version = "1.10.2" @@ -1873,6 +2335,57 @@ files = [ {file = "prospector_profile_duplicated-0.1.0.tar.gz", hash = "sha256:144baaa10101a28834d68552ff0b1714c08f22c8a7e9506e02ccc8584a28943c"}, ] +[[package]] +name = "psutil" +version = "5.9.5" +description = "Cross-platform lib for process and system monitoring in Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f"}, + {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5"}, + {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4"}, + {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"}, + {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4"}, + {file = "psutil-5.9.5-cp27-none-win32.whl", hash = "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f"}, + {file = "psutil-5.9.5-cp27-none-win_amd64.whl", hash = "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42"}, + {file = "psutil-5.9.5-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217"}, + {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da"}, + {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4"}, + {file = "psutil-5.9.5-cp36-abi3-win32.whl", hash = "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d"}, + {file = "psutil-5.9.5-cp36-abi3-win_amd64.whl", hash = "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9"}, + {file = "psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30"}, + {file = "psutil-5.9.5.tar.gz", hash = "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c"}, +] + +[package.extras] +test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] + +[[package]] +name = "ptyprocess" +version = "0.7.0" +description = "Run a subprocess in a pseudo terminal" +optional = false +python-versions = "*" +files = [ + {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, + {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, +] + +[[package]] +name = "pure-eval" +version = "0.2.2" +description = "Safely evaluate AST nodes without side effects" +optional = false +python-versions = "*" +files = [ + {file = "pure_eval-0.2.2-py3-none-any.whl", hash = "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350"}, + {file = "pure_eval-0.2.2.tar.gz", hash = "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"}, +] + +[package.extras] +tests = ["pytest"] + [[package]] name = "pycodestyle" version = "2.11.0" @@ -1888,7 +2401,7 @@ files = [ name = "pycparser" version = "2.21" description = "C parser in Python" -optional = true +optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, @@ -2244,7 +2757,7 @@ pytest = ">=6.2" name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -optional = true +optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, @@ -2295,7 +2808,7 @@ numpy = ">=1.17.3" name = "pywin32" version = "306" description = "Python for Window Extensions" -optional = true +optional = false python-versions = "*" files = [ {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, @@ -2378,11 +2891,116 @@ files = [ [package.extras] scripts = ["Pillow (>=3.2.0)"] +[[package]] +name = "pyzmq" +version = "25.1.1" +description = "Python bindings for 0MQ" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pyzmq-25.1.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:381469297409c5adf9a0e884c5eb5186ed33137badcbbb0560b86e910a2f1e76"}, + {file = "pyzmq-25.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:955215ed0604dac5b01907424dfa28b40f2b2292d6493445dd34d0dfa72586a8"}, + {file = "pyzmq-25.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:985bbb1316192b98f32e25e7b9958088431d853ac63aca1d2c236f40afb17c83"}, + {file = "pyzmq-25.1.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:afea96f64efa98df4da6958bae37f1cbea7932c35878b185e5982821bc883369"}, + {file = "pyzmq-25.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76705c9325d72a81155bb6ab48d4312e0032bf045fb0754889133200f7a0d849"}, + {file = "pyzmq-25.1.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:77a41c26205d2353a4c94d02be51d6cbdf63c06fbc1295ea57dad7e2d3381b71"}, + {file = "pyzmq-25.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:12720a53e61c3b99d87262294e2b375c915fea93c31fc2336898c26d7aed34cd"}, + {file = "pyzmq-25.1.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:57459b68e5cd85b0be8184382cefd91959cafe79ae019e6b1ae6e2ba8a12cda7"}, + {file = "pyzmq-25.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:292fe3fc5ad4a75bc8df0dfaee7d0babe8b1f4ceb596437213821f761b4589f9"}, + {file = "pyzmq-25.1.1-cp310-cp310-win32.whl", hash = "sha256:35b5ab8c28978fbbb86ea54958cd89f5176ce747c1fb3d87356cf698048a7790"}, + {file = "pyzmq-25.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:11baebdd5fc5b475d484195e49bae2dc64b94a5208f7c89954e9e354fc609d8f"}, + {file = "pyzmq-25.1.1-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:d20a0ddb3e989e8807d83225a27e5c2eb2260eaa851532086e9e0fa0d5287d83"}, + {file = "pyzmq-25.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e1c1be77bc5fb77d923850f82e55a928f8638f64a61f00ff18a67c7404faf008"}, + {file = "pyzmq-25.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d89528b4943d27029a2818f847c10c2cecc79fa9590f3cb1860459a5be7933eb"}, + {file = "pyzmq-25.1.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:90f26dc6d5f241ba358bef79be9ce06de58d477ca8485e3291675436d3827cf8"}, + {file = "pyzmq-25.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2b92812bd214018e50b6380ea3ac0c8bb01ac07fcc14c5f86a5bb25e74026e9"}, + {file = "pyzmq-25.1.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:2f957ce63d13c28730f7fd6b72333814221c84ca2421298f66e5143f81c9f91f"}, + {file = "pyzmq-25.1.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:047a640f5c9c6ade7b1cc6680a0e28c9dd5a0825135acbd3569cc96ea00b2505"}, + {file = "pyzmq-25.1.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7f7e58effd14b641c5e4dec8c7dab02fb67a13df90329e61c869b9cc607ef752"}, + {file = "pyzmq-25.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c2910967e6ab16bf6fbeb1f771c89a7050947221ae12a5b0b60f3bca2ee19bca"}, + {file = "pyzmq-25.1.1-cp311-cp311-win32.whl", hash = "sha256:76c1c8efb3ca3a1818b837aea423ff8a07bbf7aafe9f2f6582b61a0458b1a329"}, + {file = "pyzmq-25.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:44e58a0554b21fc662f2712814a746635ed668d0fbc98b7cb9d74cb798d202e6"}, + {file = "pyzmq-25.1.1-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:e1ffa1c924e8c72778b9ccd386a7067cddf626884fd8277f503c48bb5f51c762"}, + {file = "pyzmq-25.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1af379b33ef33757224da93e9da62e6471cf4a66d10078cf32bae8127d3d0d4a"}, + {file = "pyzmq-25.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cff084c6933680d1f8b2f3b4ff5bbb88538a4aac00d199ac13f49d0698727ecb"}, + {file = "pyzmq-25.1.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2400a94f7dd9cb20cd012951a0cbf8249e3d554c63a9c0cdfd5cbb6c01d2dec"}, + {file = "pyzmq-25.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d81f1ddae3858b8299d1da72dd7d19dd36aab654c19671aa8a7e7fb02f6638a"}, + {file = "pyzmq-25.1.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:255ca2b219f9e5a3a9ef3081512e1358bd4760ce77828e1028b818ff5610b87b"}, + {file = "pyzmq-25.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a882ac0a351288dd18ecae3326b8a49d10c61a68b01419f3a0b9a306190baf69"}, + {file = "pyzmq-25.1.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:724c292bb26365659fc434e9567b3f1adbdb5e8d640c936ed901f49e03e5d32e"}, + {file = "pyzmq-25.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ca1ed0bb2d850aa8471387882247c68f1e62a4af0ce9c8a1dbe0d2bf69e41fb"}, + {file = "pyzmq-25.1.1-cp312-cp312-win32.whl", hash = "sha256:b3451108ab861040754fa5208bca4a5496c65875710f76789a9ad27c801a0075"}, + {file = "pyzmq-25.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:eadbefd5e92ef8a345f0525b5cfd01cf4e4cc651a2cffb8f23c0dd184975d787"}, + {file = "pyzmq-25.1.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:db0b2af416ba735c6304c47f75d348f498b92952f5e3e8bff449336d2728795d"}, + {file = "pyzmq-25.1.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7c133e93b405eb0d36fa430c94185bdd13c36204a8635470cccc200723c13bb"}, + {file = "pyzmq-25.1.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:273bc3959bcbff3f48606b28229b4721716598d76b5aaea2b4a9d0ab454ec062"}, + {file = "pyzmq-25.1.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:cbc8df5c6a88ba5ae385d8930da02201165408dde8d8322072e3e5ddd4f68e22"}, + {file = "pyzmq-25.1.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:18d43df3f2302d836f2a56f17e5663e398416e9dd74b205b179065e61f1a6edf"}, + {file = "pyzmq-25.1.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:73461eed88a88c866656e08f89299720a38cb4e9d34ae6bf5df6f71102570f2e"}, + {file = "pyzmq-25.1.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:34c850ce7976d19ebe7b9d4b9bb8c9dfc7aac336c0958e2651b88cbd46682123"}, + {file = "pyzmq-25.1.1-cp36-cp36m-win32.whl", hash = "sha256:d2045d6d9439a0078f2a34b57c7b18c4a6aef0bee37f22e4ec9f32456c852c71"}, + {file = "pyzmq-25.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:458dea649f2f02a0b244ae6aef8dc29325a2810aa26b07af8374dc2a9faf57e3"}, + {file = "pyzmq-25.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7cff25c5b315e63b07a36f0c2bab32c58eafbe57d0dce61b614ef4c76058c115"}, + {file = "pyzmq-25.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1579413ae492b05de5a6174574f8c44c2b9b122a42015c5292afa4be2507f28"}, + {file = "pyzmq-25.1.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3d0a409d3b28607cc427aa5c30a6f1e4452cc44e311f843e05edb28ab5e36da0"}, + {file = "pyzmq-25.1.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:21eb4e609a154a57c520e3d5bfa0d97e49b6872ea057b7c85257b11e78068222"}, + {file = "pyzmq-25.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:034239843541ef7a1aee0c7b2cb7f6aafffb005ede965ae9cbd49d5ff4ff73cf"}, + {file = "pyzmq-25.1.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f8115e303280ba09f3898194791a153862cbf9eef722ad8f7f741987ee2a97c7"}, + {file = "pyzmq-25.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1a5d26fe8f32f137e784f768143728438877d69a586ddeaad898558dc971a5ae"}, + {file = "pyzmq-25.1.1-cp37-cp37m-win32.whl", hash = "sha256:f32260e556a983bc5c7ed588d04c942c9a8f9c2e99213fec11a031e316874c7e"}, + {file = "pyzmq-25.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:abf34e43c531bbb510ae7e8f5b2b1f2a8ab93219510e2b287a944432fad135f3"}, + {file = "pyzmq-25.1.1-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:87e34f31ca8f168c56d6fbf99692cc8d3b445abb5bfd08c229ae992d7547a92a"}, + {file = "pyzmq-25.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c9c6c9b2c2f80747a98f34ef491c4d7b1a8d4853937bb1492774992a120f475d"}, + {file = "pyzmq-25.1.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5619f3f5a4db5dbb572b095ea3cb5cc035335159d9da950830c9c4db2fbb6995"}, + {file = "pyzmq-25.1.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5a34d2395073ef862b4032343cf0c32a712f3ab49d7ec4f42c9661e0294d106f"}, + {file = "pyzmq-25.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25f0e6b78220aba09815cd1f3a32b9c7cb3e02cb846d1cfc526b6595f6046618"}, + {file = "pyzmq-25.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3669cf8ee3520c2f13b2e0351c41fea919852b220988d2049249db10046a7afb"}, + {file = "pyzmq-25.1.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2d163a18819277e49911f7461567bda923461c50b19d169a062536fffe7cd9d2"}, + {file = "pyzmq-25.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:df27ffddff4190667d40de7beba4a950b5ce78fe28a7dcc41d6f8a700a80a3c0"}, + {file = "pyzmq-25.1.1-cp38-cp38-win32.whl", hash = "sha256:a382372898a07479bd34bda781008e4a954ed8750f17891e794521c3e21c2e1c"}, + {file = "pyzmq-25.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:52533489f28d62eb1258a965f2aba28a82aa747202c8fa5a1c7a43b5db0e85c1"}, + {file = "pyzmq-25.1.1-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:03b3f49b57264909aacd0741892f2aecf2f51fb053e7d8ac6767f6c700832f45"}, + {file = "pyzmq-25.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:330f9e188d0d89080cde66dc7470f57d1926ff2fb5576227f14d5be7ab30b9fa"}, + {file = "pyzmq-25.1.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2ca57a5be0389f2a65e6d3bb2962a971688cbdd30b4c0bd188c99e39c234f414"}, + {file = "pyzmq-25.1.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d457aed310f2670f59cc5b57dcfced452aeeed77f9da2b9763616bd57e4dbaae"}, + {file = "pyzmq-25.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c56d748ea50215abef7030c72b60dd723ed5b5c7e65e7bc2504e77843631c1a6"}, + {file = "pyzmq-25.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8f03d3f0d01cb5a018debeb412441996a517b11c5c17ab2001aa0597c6d6882c"}, + {file = "pyzmq-25.1.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:820c4a08195a681252f46926de10e29b6bbf3e17b30037bd4250d72dd3ddaab8"}, + {file = "pyzmq-25.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17ef5f01d25b67ca8f98120d5fa1d21efe9611604e8eb03a5147360f517dd1e2"}, + {file = "pyzmq-25.1.1-cp39-cp39-win32.whl", hash = "sha256:04ccbed567171579ec2cebb9c8a3e30801723c575601f9a990ab25bcac6b51e2"}, + {file = "pyzmq-25.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:e61f091c3ba0c3578411ef505992d356a812fb200643eab27f4f70eed34a29ef"}, + {file = "pyzmq-25.1.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ade6d25bb29c4555d718ac6d1443a7386595528c33d6b133b258f65f963bb0f6"}, + {file = "pyzmq-25.1.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0c95ddd4f6e9fca4e9e3afaa4f9df8552f0ba5d1004e89ef0a68e1f1f9807c7"}, + {file = "pyzmq-25.1.1-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48e466162a24daf86f6b5ca72444d2bf39a5e58da5f96370078be67c67adc978"}, + {file = "pyzmq-25.1.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abc719161780932c4e11aaebb203be3d6acc6b38d2f26c0f523b5b59d2fc1996"}, + {file = "pyzmq-25.1.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:1ccf825981640b8c34ae54231b7ed00271822ea1c6d8ba1090ebd4943759abf5"}, + {file = "pyzmq-25.1.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:c2f20ce161ebdb0091a10c9ca0372e023ce24980d0e1f810f519da6f79c60800"}, + {file = "pyzmq-25.1.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:deee9ca4727f53464daf089536e68b13e6104e84a37820a88b0a057b97bba2d2"}, + {file = "pyzmq-25.1.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:aa8d6cdc8b8aa19ceb319aaa2b660cdaccc533ec477eeb1309e2a291eaacc43a"}, + {file = "pyzmq-25.1.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:019e59ef5c5256a2c7378f2fb8560fc2a9ff1d315755204295b2eab96b254d0a"}, + {file = "pyzmq-25.1.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:b9af3757495c1ee3b5c4e945c1df7be95562277c6e5bccc20a39aec50f826cd0"}, + {file = "pyzmq-25.1.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:548d6482dc8aadbe7e79d1b5806585c8120bafa1ef841167bc9090522b610fa6"}, + {file = "pyzmq-25.1.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:057e824b2aae50accc0f9a0570998adc021b372478a921506fddd6c02e60308e"}, + {file = "pyzmq-25.1.1-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2243700cc5548cff20963f0ca92d3e5e436394375ab8a354bbea2b12911b20b0"}, + {file = "pyzmq-25.1.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79986f3b4af059777111409ee517da24a529bdbd46da578b33f25580adcff728"}, + {file = "pyzmq-25.1.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:11d58723d44d6ed4dd677c5615b2ffb19d5c426636345567d6af82be4dff8a55"}, + {file = "pyzmq-25.1.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:49d238cf4b69652257db66d0c623cd3e09b5d2e9576b56bc067a396133a00d4a"}, + {file = "pyzmq-25.1.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fedbdc753827cf014c01dbbee9c3be17e5a208dcd1bf8641ce2cd29580d1f0d4"}, + {file = "pyzmq-25.1.1-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bc16ac425cc927d0a57d242589f87ee093884ea4804c05a13834d07c20db203c"}, + {file = "pyzmq-25.1.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11c1d2aed9079c6b0c9550a7257a836b4a637feb334904610f06d70eb44c56d2"}, + {file = "pyzmq-25.1.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e8a701123029cc240cea61dd2d16ad57cab4691804143ce80ecd9286b464d180"}, + {file = "pyzmq-25.1.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:61706a6b6c24bdece85ff177fec393545a3191eeda35b07aaa1458a027ad1304"}, + {file = "pyzmq-25.1.1.tar.gz", hash = "sha256:259c22485b71abacdfa8bf79720cd7bcf4b9d128b30ea554f01ae71fdbfdaa23"}, +] + +[package.dependencies] +cffi = {version = "*", markers = "implementation_name == \"pypy\""} + [[package]] name = "referencing" version = "0.30.0" description = "JSON Referencing + Python" -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "referencing-0.30.0-py3-none-any.whl", hash = "sha256:c257b08a399b6c2f5a3510a50d28ab5dbc7bbde049bcaf954d43c446f83ab548"}, @@ -2472,7 +3090,7 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] name = "rpds-py" version = "0.9.2" description = "Python bindings to Rust's persistent data structures (rpds)" -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "rpds_py-0.9.2-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:ab6919a09c055c9b092798ce18c6c4adf49d24d4d9e43a92b257e3f2548231e7"}, @@ -2798,6 +3416,36 @@ files = [ {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, ] +[[package]] +name = "soupsieve" +version = "2.4.1" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "soupsieve-2.4.1-py3-none-any.whl", hash = "sha256:1c1bfee6819544a3447586c889157365a27e10d88cde3ad3da0cf0ddf646feb8"}, + {file = "soupsieve-2.4.1.tar.gz", hash = "sha256:89d12b2d5dfcd2c9e8c22326da9d9aa9cb3dfab0a83a024f05704076ee8d35ea"}, +] + +[[package]] +name = "stack-data" +version = "0.6.2" +description = "Extract data from python stack frames and tracebacks for informative displays" +optional = false +python-versions = "*" +files = [ + {file = "stack_data-0.6.2-py3-none-any.whl", hash = "sha256:cbb2a53eb64e5785878201a97ed7c7b94883f48b87bfb0bbe8b623c74679e4a8"}, + {file = "stack_data-0.6.2.tar.gz", hash = "sha256:32d2dd0376772d01b6cb9fc996f3c8b57a357089dec328ed4b6553d037eaf815"}, +] + +[package.dependencies] +asttokens = ">=2.1.0" +executing = ">=1.2.0" +pure-eval = "*" + +[package.extras] +tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] + [[package]] name = "stevedore" version = "5.1.0" @@ -2833,7 +3481,7 @@ all = ["defusedxml", "fsspec", "imagecodecs (>=2023.1.23)", "lxml", "matplotlib" name = "tinycss2" version = "1.2.1" description = "A tiny CSS parser" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "tinycss2-1.2.1-py3-none-any.whl", hash = "sha256:2b80a96d41e7c3914b8cda8bc7f705a4d9c49275616e886103dd839dfc847847"}, @@ -2880,11 +3528,31 @@ files = [ {file = "tomlkit-0.12.1.tar.gz", hash = "sha256:38e1ff8edb991273ec9f6181244a6a391ac30e9f5098e7535640ea6be97a7c86"}, ] +[[package]] +name = "tornado" +version = "6.3.3" +description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +optional = false +python-versions = ">= 3.8" +files = [ + {file = "tornado-6.3.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:502fba735c84450974fec147340016ad928d29f1e91f49be168c0a4c18181e1d"}, + {file = "tornado-6.3.3-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:805d507b1f588320c26f7f097108eb4023bbaa984d63176d1652e184ba24270a"}, + {file = "tornado-6.3.3-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bd19ca6c16882e4d37368e0152f99c099bad93e0950ce55e71daed74045908f"}, + {file = "tornado-6.3.3-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ac51f42808cca9b3613f51ffe2a965c8525cb1b00b7b2d56828b8045354f76a"}, + {file = "tornado-6.3.3-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71a8db65160a3c55d61839b7302a9a400074c9c753040455494e2af74e2501f2"}, + {file = "tornado-6.3.3-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ceb917a50cd35882b57600709dd5421a418c29ddc852da8bcdab1f0db33406b0"}, + {file = "tornado-6.3.3-cp38-abi3-musllinux_1_1_i686.whl", hash = "sha256:7d01abc57ea0dbb51ddfed477dfe22719d376119844e33c661d873bf9c0e4a16"}, + {file = "tornado-6.3.3-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:9dc4444c0defcd3929d5c1eb5706cbe1b116e762ff3e0deca8b715d14bf6ec17"}, + {file = "tornado-6.3.3-cp38-abi3-win32.whl", hash = "sha256:65ceca9500383fbdf33a98c0087cb975b2ef3bfb874cb35b8de8740cf7f41bd3"}, + {file = "tornado-6.3.3-cp38-abi3-win_amd64.whl", hash = "sha256:22d3c2fa10b5793da13c807e6fc38ff49a4f6e1e3868b0a6f4164768bb8e20f5"}, + {file = "tornado-6.3.3.tar.gz", hash = "sha256:e7d8db41c0181c80d76c982aacc442c0783a2c54d6400fe028954201a2e032fe"}, +] + [[package]] name = "traitlets" version = "5.9.0" description = "Traitlets Python configuration system" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "traitlets-5.9.0-py3-none-any.whl", hash = "sha256:9e6ec080259b9a5940c797d58b613b5e31441c2257b87c2e795c5228ae80d2d8"}, @@ -2988,6 +3656,17 @@ files = [ docs = ["Sphinx", "repoze.sphinx.autointerface"] testing = ["coverage", "pytest", "pytest-cov"] +[[package]] +name = "wcwidth" +version = "0.2.6" +description = "Measures the displayed width of unicode strings in a terminal" +optional = false +python-versions = "*" +files = [ + {file = "wcwidth-0.2.6-py2.py3-none-any.whl", hash = "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e"}, + {file = "wcwidth-0.2.6.tar.gz", hash = "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"}, +] + [[package]] name = "weasyprint" version = "59.0" @@ -3017,7 +3696,7 @@ test = ["flake8", "isort", "pytest"] name = "webencodings" version = "0.5.1" description = "Character encoding aliases for legacy web content" -optional = true +optional = false python-versions = "*" files = [ {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, @@ -3127,7 +3806,7 @@ files = [ name = "zipp" version = "3.16.2" description = "Backport of pathlib-compatible object wrapper for zip files" -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "zipp-3.16.2-py3-none-any.whl", hash = "sha256:679e51dd4403591b2d6838a48de3d283f3d188412a9782faadf845f298736ba0"}, @@ -3311,4 +3990,4 @@ process = ["Jinja2", "Pillow", "PyPDF2", "cffi", "deepmerge", "deskew", "matplot [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "1d2534d1114249baa7d4ed7d847fcb6c024fc4d46439490316a003eaf4e25a21" +content-hash = "75db5d40ca60d03d5ce7c813d655f87b061da2faf133b14e3ffe6ed4f0dc2428" diff --git a/pyproject.toml b/pyproject.toml index 5c12827d..165cd1f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,10 @@ line-length = 110 target-version = ["py39"] +[tool.isort] +profile = "black" +line_length = 110 + [tool.mypy] python_version = 3.9 ignore_missing_imports = true @@ -97,6 +101,9 @@ pyroma = "4.2" typing-extensions = "4.7.1" c2cwsgiutils = { version = "6.0.0.dev142", extras = ["test_images"] } types-requests = "2.31.0.2" +nbconvert = "7.7.3" +ipykernel = "6.25.1" + [build-system] requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning", "poetry-plugin-tweak-dependencies-version"] diff --git a/scan_to_paperless/__init__.py b/scan_to_paperless/__init__.py index 83f8d31d..51ffab80 100644 --- a/scan_to_paperless/__init__.py +++ b/scan_to_paperless/__init__.py @@ -23,6 +23,10 @@ CONFIG_PATH = os.path.join(CONFIG_FOLDER, CONFIG_FILENAME) +class ScanToPaperlessException(Exception): + """Base exception for this module.""" + + def get_config(config_filename: str) -> schema.Configuration: """Get the configuration.""" if os.path.exists(config_filename): diff --git a/scan_to_paperless/jupyter.py b/scan_to_paperless/jupyter.py new file mode 100644 index 00000000..800810ab --- /dev/null +++ b/scan_to_paperless/jupyter.py @@ -0,0 +1,388 @@ +"""Functions used to generate a Jupyter notebook from the transform.""" + + +import os +from typing import TYPE_CHECKING, Any + +# read, write, rotate, crop, sharpen, draw_line, find_line, find_contour +import nbformat +import numpy as np + +import scan_to_paperless +import scan_to_paperless.process_utils +from scan_to_paperless import process_schema as schema + +if TYPE_CHECKING: + NpNdarrayInt = np.ndarray[np.uint8, Any] +else: + NpNdarrayInt = np.ndarray + + +def _pretty_repr(value: Any, prefix: str = "") -> str: + if isinstance(value, dict): + return "\n".join( + [ + "{", + *[ + f'{prefix} "{key}": {_pretty_repr(value, prefix + " ")},' + for key, value in value.items() + ], + prefix + "}", + ] + ) + + return repr(value) + + +def create_transform_notebook( + root_folder: str, context: scan_to_paperless.process_utils.Context, step: schema.Step +) -> None: + """Create a Jupyter notebook for the transform step.""" + + # Jupyter notebook + dest_folder = os.path.join(root_folder, "jupyter") + if not os.path.exists(dest_folder): + os.makedirs(dest_folder) + with open(os.path.join(dest_folder, "README.txt"), "w", encoding="utf-8") as readme_file: + readme_file.write( + """# Jupyter notebook + +Install dependencies: +pip install scan-to-paperless[process] jupyterlab Pillow + +Run: +jupyter lab + +Open the notebook file. +""" + ) + + notebook = nbformat.v4.new_notebook() # type: ignore[no-untyped-call] + + notebook["cells"].append( + nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] + """# Scan to Paperless + +This notebook show the transformation applied on the images of the document. + +At the start of each step, se set some values on the `context.config["args"]` dict, +you can change the values to see the impact on the result, +then yon can all those changes in the `config.yaml` file, in the `args` section.""" + ) + ) + + notebook["cells"].append( + nbformat.v4.new_markdown_cell("Do the required imports.") # type: ignore[no-untyped-call] + ) + notebook["cells"].append( + nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] + """import os +import cv2 +import numpy as np + +from scan_to_paperless import process, process_utils""" + ) + ) + + notebook["cells"].append( + nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] + """Calculate the base folder of the document.""" + ) + ) + notebook["cells"].append( + nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] + """import IPython + +jupyter_locals = IPython.extract_module_locals()[1] +base_folder = os.path.dirname(os.path.dirname(jupyter_locals['__vsc_ipynb_file__']) if '' in jupyter_locals else os.getcwd())""" + ) + ) + + notebook["cells"].append( + nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] + """Open on of the source images, you can change it by uncommenting the corresponding line.""" + ) + ) + other_images_open = "\n".join( + [ + f'# context.image = cv2.imread(os.path.join(base_folder, "{image}"))' + for image in step["sources"][1:] + ] + ) + notebook["cells"].append( + nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] + f"""# Open Source image +context = process_utils.Context({{"args": {{}}}}, {{}}) + +# Open one of the images +context.image = cv2.imread(os.path.join(base_folder, "{step["sources"][0]}")) +{other_images_open} + +images_context = {{"original": context.image}}""" + ) + ) + + notebook["cells"].append( + nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] + """Set the values that's used by more than one step.""" + ) + ) + notebook["cells"].append( + nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] + f"""context.config["args"] = {{ + "dpi": {context.config["args"].get("dpi", schema.DPI_DEFAULT)}, + "background_color": {context.config["args"].get("background_color", schema.BACKGROUND_COLOR_DEFAULT)}, +}}""" + ) + ) + + notebook["cells"].append( + nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] + """Get the index that represent the part of the image we want to see.""" + ) + ) + notebook["cells"].append( + nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] + """ +# Get a part of the image to display, by default, the top of the image +context.get_index = lambda image: np.ix_( + np.arange(0, min(image.shape[0], 500)), + np.arange(0, image.shape[1]), + np.arange(0, image.shape[2]), +) + +context.display_image(images_context["original"])""" + ) + ) + + notebook["cells"].append( + nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] + """Calculate the image mask, the mask is used to hide some part of the image when we calculate the image skew and the image auto crop (based on the content). + +The `lower_hsv_color`and the `upper_hsv_color` are used to define the color range to remove, +the `de_noise_size` is used to remove noise from the image, +the `buffer_size` is used to add a buffer around the image and +the `buffer_level` is used to define the level of the buffer (`0.0` to `1.0`). + +To remove the gray background from the scanner, on document I use the following values: +```yaml +lower_hsv_color: [0, 0, 250] +upper_hsv_color: [255, 10, 255] +``` +On leaflet I use the following values: +```yaml +lower_hsv_color: [0, 20, 0] +upper_hsv_color: [255, 255, 255] +``` +""" + ) + ) + notebook["cells"].append( + nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] + f"""context.image = images_context["original"].copy() + +context.config["args"]["auto_mask"] = {_pretty_repr(context.config["args"].get("auto_mask", {}))} + +hsv = cv2.cvtColor(context.image, cv2.COLOR_BGR2HSV) +print("Hue (h)") +context.display_image(cv2.cvtColor(hsv[:, :, 0], cv2.COLOR_GRAY2RGB)) +print("Saturation (s)") +context.display_image(cv2.cvtColor(hsv[:, :, 1], cv2.COLOR_GRAY2RGB)) +print("Value (v)") +context.display_image(cv2.cvtColor(hsv[:, :, 2], cv2.COLOR_GRAY2RGB)) + +# Print the HSV value on some point of the image +points = [ + [10, 10], + [100, 100], +] +image = context.image.copy() +for x, y in points: + print(f"Pixel: {{x}}:{{y}}, with value: {{hsv[y, x, :]}}") + cv2.drawMarker(image, [x, y], (0, 0, 255), cv2.MARKER_CROSS, 20, 2) +context.display_image(image) + +context.init_mask() +if context.mask is not None: + context.display_image(cv2.cvtColor(context.mask, cv2.COLOR_GRAY2RGB)) +context.display_image(context.get_masked()) + +images_context["auto_mask"] = context.image""" + ) + ) + + notebook["cells"].append( + nbformat.v4.new_markdown_cell("Display the image histogram.") # type: ignore[no-untyped-call] + ) + notebook["cells"].append( + nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] + f"""context.image = images_context["auto_mask"].copy() + +context.config["args"]["level"] = {context.config["args"].get("level", schema.LEVEL_DEFAULT)} +context.config["args"]["cut_white"] = {context.config["args"].get("cut_white", schema.CUT_WHITE_DEFAULT)} +context.config["args"]["cut_black"] = {context.config["args"].get("cut_black", schema.CUT_BLACK_DEFAULT)} + +process.histogram(context)""" + ) + ) + + notebook["cells"].append( + nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] + """Do the image level correction. + +Some of the used values are displayed in the histogram chart.""" + ) + ) + notebook["cells"].append( + nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] + f"""context.image = images_context["auto_mask"].copy() + +context.config["args"]["auto_level"] = {context.config["args"].get("auto_level", schema.AUTO_LEVEL_DEFAULT)}, +context.config["args"]["level"] = {context.config["args"].get("level", schema.LEVEL_DEFAULT)}, + +process.level(context) +context.display_image(context.image) + +images_context["level"] = context.image""" + ) + ) + + notebook["cells"].append( + nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] + """Do the image level cut correction. + +Some of the used values are displayed in the histogram chart.""" + ) + ) + notebook["cells"].append( + nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] + f"""context.image = images_context["level"].copy() + +print(f"Use cut_white: {context.config["args"]["cut_white"]}") +print(f"Use cut_black: {context.config["args"]["cut_black"]}") + +process.color_cut(context) +context.display_image(context.image) + +images_context["color_cut"] = context.image""" + ) + ) + + notebook["cells"].append( + nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] + """Cut some part of the image by auto removing a part of the image. + +The needed of this step is to remove some part of the image that represent the part that is out of the page, witch is gray with some scanner. + +The `lower_hsv_color`and the `upper_hsv_color` are used to define the color range to remove, +the `de_noise_size` is used to remove noise from the image, +the `buffer_size` is used to add a buffer around the image and +the `buffer_level` is used to define the level of the buffer (`0.0` to `1.0`).""" + ) + ) + notebook["cells"].append( + nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] + f"""context.image = images_context["color_cut"].copy() + +context.config["args"]["auto_cut"] = {_pretty_repr(context.config["args"].get("auto_cut", {}))} + +# Print in HSV some point of the image +hsv = cv2.cvtColor(context.image, cv2.COLOR_BGR2HSV) +print("Pixel 10:10: ", hsv[10, 10]) +print("Pixel 100:100: ", hsv[100, 100]) + +process.cut(context) +context.display_image(context.image) + +images_context["cut"] = context.image""" + ) + ) + + notebook["cells"].append( + nbformat.v4.new_markdown_cell("Do the image skew correction.") # type: ignore[no-untyped-call] + ) + notebook["cells"].append( + nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] + f"""context.image = images_context["cut"].copy() + +context.config["args"]["deskew"] = {_pretty_repr(context.config["args"].get("deskew", {}))} + +# The angle can be forced in config.images_config..angle. +process.deskew(context) +context.display_image(context.image) + +images_context["deskew"] = context.image""" + ) + ) + + notebook["cells"].append( + nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] + """Do the image auto crop base on the image content.""" + ) + ) + notebook["cells"].append( + nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] + f"""context.image = images_context["deskew"].copy() + +context.config["args"]["crop"] = {_pretty_repr(context.config["args"].get("crop", {}))} + +process.docrop(context) +context.display_image(context.image) + +images_context["crop"] = context.image""" + ) + ) + + notebook["cells"].append( + nbformat.v4.new_markdown_cell("Do the image sharpen correction.") # type: ignore[no-untyped-call] + ) + notebook["cells"].append( + nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] + f"""context.image = images_context["crop"].copy() + +context.config["args"]["sharpen"] = {context.config["args"].get("sharpen", schema.SHARPEN_DEFAULT)} + +process.sharpen(context) +context.display_image(context.image) + +images_context["sharpen"] = context.image""" + ) + ) + + notebook["cells"].append( + nbformat.v4.new_markdown_cell("Do the image dither correction.") # type: ignore[no-untyped-call] + ) + notebook["cells"].append( + nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] + f"""context.image = images_context["sharpen"].copy() + +context.config["args"]["dither"] = {context.config["args"].get("dither", schema.DITHER_DEFAULT)} + +process.dither(context) +context.display_image(context.image) + +images_context["dither"] = context.image""" + ) + ) + + notebook["cells"].append( + nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] + """Do the image auto rotate correction, based on the text orientation. + +This require Tesseract to be installed.""" + ) + ) + notebook["cells"].append( + nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] + """context.image = images_context["dither"].copy() + +try: + process.autorotate(context) + context.display_image(context.image) +except FileNotFoundError as e: + print("Tesseract not found, skipping autorotate: ", e)""" + ) + ) + + with open(os.path.join(dest_folder, "jupyter.ipynb"), "w", encoding="utf-8") as jupyter_file: + nbformat.write(notebook, jupyter_file) # type: ignore[no-untyped-call] diff --git a/scan_to_paperless/jupyter_utils.py b/scan_to_paperless/jupyter_utils.py new file mode 100644 index 00000000..1deeaf8c --- /dev/null +++ b/scan_to_paperless/jupyter_utils.py @@ -0,0 +1,11 @@ +"""Utility function related to Jupyter notebook.""" + + +def is_ipython() -> bool: + """Return True if running in IPython (Jupyter).""" + + try: + __IPYTHON__ # type: ignore[name-defined] # pylint: disable=pointless-statement + return True + except NameError: + return False diff --git a/scan_to_paperless/process.py b/scan_to_paperless/process.py index 84576adb..e1e3599e 100755 --- a/scan_to_paperless/process.py +++ b/scan_to_paperless/process.py @@ -6,7 +6,6 @@ import datetime import json import logging -import math import os import re import shutil @@ -15,12 +14,11 @@ import tempfile import time import traceback -from typing import IO, TYPE_CHECKING, Any, Callable, Optional, Protocol, TypedDict, Union, cast +from typing import IO, TYPE_CHECKING, Any, Optional, Protocol, TypedDict, Union, cast # read, write, rotate, crop, sharpen, draw_line, find_line, find_contour import cv2 import matplotlib.pyplot as plt -import nbformat import numpy as np import pikepdf import requests @@ -35,8 +33,9 @@ import scan_to_paperless import scan_to_paperless.status -from scan_to_paperless import code +from scan_to_paperless import code, jupyter_utils from scan_to_paperless import process_schema as schema +from scan_to_paperless import process_utils if TYPE_CHECKING: NpNdarrayInt = np.ndarray[np.uint8, Any] @@ -50,323 +49,6 @@ _LOG = logging.getLogger(__name__) -class ScanToPaperlessException(Exception): - """Base exception for this module.""" - - -def rotate_image( - image: NpNdarrayInt, angle: float, background: Union[int, tuple[int, int, int]] -) -> NpNdarrayInt: - """Rotate the image.""" - - old_width, old_height = image.shape[:2] - angle_radian = math.radians(angle) - width = abs(np.sin(angle_radian) * old_height) + abs(np.cos(angle_radian) * old_width) - height = abs(np.sin(angle_radian) * old_width) + abs(np.cos(angle_radian) * old_height) - - image_center: tuple[Any, ...] = tuple(np.array(image.shape[1::-1]) / 2) - rot_mat = cv2.getRotationMatrix2D(image_center, angle, 1.0) - rot_mat[1, 2] += (width - old_width) / 2 - rot_mat[0, 2] += (height - old_height) / 2 - return cast( - NpNdarrayInt, - cv2.warpAffine(image, rot_mat, (int(round(height)), int(round(width))), borderValue=background), - ) - - -def crop_image( # pylint: disable=too-many-arguments - image: NpNdarrayInt, - x: int, - y: int, - width: int, - height: int, - background: Union[tuple[int], tuple[int, int, int]], -) -> NpNdarrayInt: - """Crop the image.""" - - matrice: NpNdarrayInt = np.array([[1.0, 0.0, -x], [0.0, 1.0, -y]]) - return cast( - NpNdarrayInt, - cv2.warpAffine(image, matrice, (int(round(width)), int(round(height))), borderValue=background), - ) - - -class Context: # pylint: disable=too-many-instance-attributes - """All the context of the current image with his mask.""" - - def __init__( # pylint: disable=too-many-arguments - self, - config: schema.Configuration, - step: schema.Step, - config_file_name: Optional[str] = None, - root_folder: Optional[str] = None, - image_name: Optional[str] = None, - ) -> None: - """Initialize.""" - - self.config = config - self.step = step - self.config_file_name = config_file_name - self.root_folder = root_folder - self.image_name = image_name - self.image: Optional[NpNdarrayInt] = None - self.mask: Optional[NpNdarrayInt] = None - self.get_index: Callable[ - [NpNdarrayInt], Optional[tuple[np.ndarray[Any, np.dtype[np.signedinteger[Any]]], ...]] - ] = lambda image: np.ix_( - np.arange(0, image.shape[1]), - np.arange(0, image.shape[1]), - np.arange(0, image.shape[2]), - ) - - self.process_count = self.step.get("process_count", 0) - - def _get_mask( - self, - auto_mask_config: Optional[schema.AutoMask], - config_section: str, - default_file_name: str, - ) -> Optional[NpNdarrayInt]: - """Init the mask.""" - - if auto_mask_config is not None: - hsv = cv2.cvtColor(self.image, cv2.COLOR_BGR2HSV) - - lower_val = np.array( - auto_mask_config.setdefault("lower_hsv_color", schema.LOWER_HSV_COLOR_DEFAULT) - ) - upper_val = np.array( - auto_mask_config.setdefault("upper_hsv_color", schema.UPPER_HSV_COLOR_DEFAULT) - ) - mask = cv2.inRange(hsv, lower_val, upper_val) - - de_noise_size = auto_mask_config.setdefault("de_noise_size", schema.DE_NOISE_SIZE_DEFAULT) - mask = cv2.copyMakeBorder( - mask, - de_noise_size, - de_noise_size, - de_noise_size, - de_noise_size, - cv2.BORDER_REPLICATE, - ) - if auto_mask_config.setdefault("de_noise_morphology", schema.DE_NOISE_MORPHOLOGY_DEFAULT): - mask = cv2.morphologyEx( - mask, - cv2.MORPH_CLOSE, - cv2.getStructuringElement(cv2.MORPH_RECT, (de_noise_size, de_noise_size)), - ) - else: - blur = cv2.blur( - mask, - (de_noise_size, de_noise_size), - ) - _, mask = cv2.threshold( - blur, - auto_mask_config.setdefault("de_noise_level", schema.DE_NOISE_LEVEL_DEFAULT), - 255, - cv2.THRESH_BINARY, - ) - - inverse_mask = auto_mask_config.setdefault("inverse_mask", schema.INVERSE_MASK_DEFAULT) - if not inverse_mask: - mask = cv2.bitwise_not(mask) - - buffer_size = auto_mask_config.setdefault("buffer_size", schema.BUFFER_SIZE_DEFAULT) - blur = cv2.blur(mask, (buffer_size, buffer_size)) - _, mask = cv2.threshold( - blur, - auto_mask_config.setdefault("buffer_level", schema.BUFFER_LEVEL_DEFAULT), - 255, - cv2.THRESH_BINARY, - ) - - mask = mask[de_noise_size:-de_noise_size, de_noise_size:-de_noise_size] - - if self.root_folder: - mask_file: str = os.path.join(self.root_folder, default_file_name) - assert mask_file - if not os.path.exists(mask_file): - base_folder = os.path.dirname(self.root_folder) - assert base_folder - mask_file = os.path.join(base_folder, default_file_name) - if not os.path.exists(mask_file): - mask_file = "" - mask_file = auto_mask_config.setdefault("additional_filename", mask_file) - if mask_file and os.path.exists(mask_file): - mask = cv2.add( - mask, - cv2.bitwise_not( - cv2.resize( - cv2.imread(mask_file, cv2.IMREAD_GRAYSCALE), - (mask.shape[1], mask.shape[0]), - ) - ), - ) - - final_mask = cv2.bitwise_not(mask) - - if os.environ.get("PROGRESS", "FALSE") == "TRUE" and self.root_folder: - self.save_progress_images(config_section.replace("_", "-"), final_mask) - elif self.root_folder: - mask_file = os.path.join(self.root_folder, default_file_name) - if not os.path.exists(mask_file): - base_folder = os.path.dirname(self.root_folder) - assert base_folder - mask_file = os.path.join(base_folder, default_file_name) - if not os.path.exists(mask_file): - return None - - final_mask = cv2.imread(mask_file, cv2.IMREAD_GRAYSCALE) - if self.image is not None and final_mask is not None: - return cast(NpNdarrayInt, cv2.resize(final_mask, (self.image.shape[1], self.image.shape[0]))) - return cast(NpNdarrayInt, final_mask) - - def init_mask(self) -> None: - """Init the mask image used to mask the image on the crop and skew calculation.""" - - auto_mask_config = self.config["args"].setdefault( - "auto_mask", cast(schema.AutoMaskOperation, schema.AUTO_MASK_OPERATION_DEFAULT) - ) - self.mask = ( - self._get_mask( - auto_mask_config.setdefault("auto_mask", {}), - "auto_mask", - "mask.png", - ) - if auto_mask_config.setdefault("enabled", schema.AUTO_MASK_ENABLED_DEFAULT) - else None - ) - - def get_background_color(self) -> tuple[int, int, int]: - """Get the background color.""" - - return cast( - tuple[int, int, int], - self.config["args"].setdefault("background_color", schema.BACKGROUND_COLOR_DEFAULT), - ) - - def do_initial_cut(self) -> None: - """Definitively mask the original image.""" - - if "auto_cut" in self.config["args"]: - assert self.image is not None - mask = self._get_mask( - self.config["args"] - .setdefault("auto_cut", cast(schema.AutoCut, schema.AUTO_CUT_DEFAULT)) - .setdefault("auto_mask", {}), - "auto_cut", - "cut.png", - ) - self.image[mask == 0] = self.get_background_color() - - def get_process_count(self) -> int: - """Get the step number.""" - - try: - return self.process_count - finally: - self.process_count += 1 - - def get_masked(self) -> NpNdarrayInt: - """Get the mask.""" - - if self.image is None: - raise ScanToPaperlessException("The image is None") - if self.mask is None: - return self.image.copy() - - image = self.image.copy() - image[self.mask == 0] = self.get_background_color() - return image - - def crop(self, x: int, y: int, width: int, height: int) -> None: - """Crop the image.""" - - if self.image is None: - raise ScanToPaperlessException("The image is None") - self.image = crop_image(self.image, x, y, width, height, self.get_background_color()) - if self.mask is not None: - self.mask = crop_image(self.mask, x, y, width, height, (0,)) - - def rotate(self, angle: float) -> None: - """Rotate the image.""" - - if self.image is None: - raise ScanToPaperlessException("The image is None") - self.image = rotate_image(self.image, angle, self.get_background_color()) - if self.mask is not None: - self.mask = rotate_image(self.mask, angle, 0) - - def get_px_value(self, value: Union[int, float]) -> float: - """Get the value in px.""" - - return value / 10 / 2.51 * self.config["args"].setdefault("dpi", schema.DPI_DEFAULT) - - def is_progress(self) -> bool: - """Return we want to have the intermediate files.""" - - return os.environ.get("PROGRESS", "FALSE") == "TRUE" or self.config.setdefault( - "progress", schema.PROGRESS_DEFAULT - ) - - def save_progress_images( - self, - name: str, - image: Optional[NpNdarrayInt] = None, - image_prefix: str = "", - process_count: Optional[int] = None, - force: bool = False, - ) -> Optional[str]: - """Save the intermediate images.""" - - if _is_ipython(): - if image is None: - return None - - from IPython.display import display # pylint: disable=import-outside-toplevel,import-error - - display(Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))) - return None - - if process_count is None: - process_count = self.get_process_count() - if (self.is_progress() or force) and self.image_name is not None and self.root_folder is not None: - name = f"{process_count}-{name}" if self.is_progress() else name - dest_folder = os.path.join(self.root_folder, name) - if not os.path.exists(dest_folder): - os.makedirs(dest_folder) - dest_image = os.path.join(dest_folder, image_prefix + self.image_name) - if image is not None: - try: - cv2.imwrite(dest_image, image) - return dest_image - except Exception as exception: - print(exception) - else: - try: - cv2.imwrite(dest_image, self.image) - except Exception as exception: - print(exception) - dest_image = os.path.join(dest_folder, "mask-" + self.image_name) - try: - dest_image = os.path.join(dest_folder, "masked-" + self.image_name) - except Exception as exception: - print(exception) - try: - cv2.imwrite(dest_image, self.get_masked()) - except Exception as exception: - print(exception) - return None - - def display_image(self, image: NpNdarrayInt) -> None: - """Display the image.""" - - if _is_ipython(): - from IPython.display import display # pylint: disable=import-outside-toplevel,import-error - - display(Image.fromarray(cv2.cvtColor(image[self.get_index(image)], cv2.COLOR_BGR2RGB))) - - def add_intermediate_error( config: schema.Configuration, config_file_name: Optional[str], @@ -376,7 +58,7 @@ def add_intermediate_error( """Add in the config non fatal error.""" if config_file_name is None: - raise ScanToPaperlessException("The config file name is required") from error + raise scan_to_paperless.ScanToPaperlessException("The config file name is required") from error if "intermediate_error" not in config: config["intermediate_error"] = [] @@ -451,21 +133,21 @@ def image_diff(image1: NpNdarrayInt, image2: NpNdarrayInt) -> tuple[float, NpNda class FunctionWithContextReturnsImage(Protocol): """Function with context and returns an image.""" - def __call__(self, context: Context) -> Optional[NpNdarrayInt]: + def __call__(self, context: process_utils.Context) -> Optional[NpNdarrayInt]: """Call the function.""" class FunctionWithContextReturnsNone(Protocol): """Function with context and no return.""" - def __call__(self, context: Context) -> None: + def __call__(self, context: process_utils.Context) -> None: """Call the function.""" class ExternalFunction(Protocol): """Function that call an external tool.""" - def __call__(self, context: Context, source: str, destination: str) -> None: + def __call__(self, context: process_utils.Context, source: str, destination: str) -> None: """Call the function.""" @@ -486,7 +168,7 @@ def __init__(self, name: str, ignore_error: bool = False, progress: bool = True) def __call__(self, func: FunctionWithContextReturnsImage) -> FunctionWithContextReturnsNone: """Call the function.""" - def wrapper(context: Context) -> None: + def wrapper(context: process_utils.Context) -> None: start_time = time.perf_counter() if self.ignore_error: try: @@ -495,7 +177,7 @@ def wrapper(context: Context) -> None: context.image = new_image except Exception as exception: print(exception) - if not _is_ipython(): + if not jupyter_utils.is_ipython(): add_intermediate_error( context.config, context.config_file_name, @@ -519,7 +201,7 @@ def wrapper(context: Context) -> None: def external(func: ExternalFunction) -> FunctionWithContextReturnsImage: """Run an external tool.""" - def wrapper(context: Context) -> Optional[NpNdarrayInt]: + def wrapper(context: process_utils.Context) -> Optional[NpNdarrayInt]: with tempfile.NamedTemporaryFile(suffix=".png") as source: cv2.imwrite(source.name, context.image) with tempfile.NamedTemporaryFile(suffix=".png") as destination: @@ -554,7 +236,7 @@ def get_contour_to_crop( ) -def crop(context: Context, margin_horizontal: int, margin_vertical: int) -> None: +def crop(context: process_utils.Context, margin_horizontal: int, margin_vertical: int) -> None: """ Do a crop on an image. @@ -575,7 +257,7 @@ def crop(context: Context, margin_horizontal: int, margin_vertical: int) -> None draw_rectangle(image, contour) context.save_progress_images( "crop", - image[context.get_index(image)] if _is_ipython() else image, + image[context.get_index(image)] if jupyter_utils.is_ipython() else image, process_count=process_count, force=True, ) @@ -584,7 +266,7 @@ def crop(context: Context, margin_horizontal: int, margin_vertical: int) -> None context.crop(x, y, width, height) -def _get_level(context: Context) -> tuple[bool, float, float]: +def _get_level(context: process_utils.Context) -> tuple[bool, float, float]: level_ = context.config["args"].setdefault("level", schema.LEVEL_DEFAULT) min_p100 = 0.0 max_p100 = 100.0 @@ -604,7 +286,7 @@ def _get_level(context: Context) -> tuple[bool, float, float]: def _histogram( - context: Context, + context: process_utils.Context, histogram_data: Any, histogram_centers: Any, histogram_max: Any, @@ -655,7 +337,7 @@ def _histogram( plt.tight_layout() with tempfile.NamedTemporaryFile(suffix=".png") as file: - if not _is_ipython(): + if not jupyter_utils.is_ipython(): plt.savefig(file.name) subprocess.run(["gm", "convert", "-flatten", file.name, file.name], check=True) # nosec image = cv2.imread(file.name) @@ -669,7 +351,7 @@ def _histogram( @Process("histogram", progress=False) -def histogram(context: Context) -> None: +def histogram(context: process_utils.Context) -> None: """Create an image with the histogram of the current image.""" noisy_image = img_as_ubyte(context.image) @@ -682,7 +364,7 @@ def histogram(context: Context) -> None: @Process("level") -def level(context: Context) -> NpNdarrayInt: +def level(context: process_utils.Context) -> NpNdarrayInt: """Do the level on an image.""" img_yuv = cv2.cvtColor(context.image, cv2.COLOR_BGR2YUV) @@ -703,7 +385,7 @@ def level(context: Context) -> NpNdarrayInt: @Process("color-cut") -def color_cut(context: Context) -> None: +def color_cut(context: process_utils.Context) -> None: """Set the near white to white and near black to black.""" assert context.image is not None @@ -720,14 +402,14 @@ def color_cut(context: Context) -> None: @Process("mask-cut") -def cut(context: Context) -> None: +def cut(context: process_utils.Context) -> None: """Mask the image with the cut mask.""" context.do_initial_cut() @Process("deskew") -def deskew(context: Context) -> None: +def deskew(context: process_utils.Context) -> None: """Deskew an image.""" images_config = context.config.setdefault("images_config", {}) @@ -755,7 +437,7 @@ def deskew(context: Context) -> None: image_status["angle"] = float(skew_angle) angle = float(skew_angle) - if not _is_ipython(): + if not jupyter_utils.is_ipython(): process_count = context.get_process_count() for name, image in debug_images: context.save_progress_images("skew", image, name, process_count, True) @@ -768,7 +450,7 @@ def deskew(context: Context) -> None: sources = [img for img in context.config.get("images", []) if f"{image_name_split[0]}." in img] if len(sources) == 1: assert context.root_folder - image = rotate_image( + image = process_utils.rotate_image( cv2.imread(os.path.join(context.root_folder, sources[0])), angle, context.get_background_color(), @@ -781,7 +463,7 @@ def deskew(context: Context) -> None: @Process("docrop") -def docrop(context: Context) -> None: +def docrop(context: process_utils.Context) -> None: """Crop an image.""" # Margin in mm @@ -798,7 +480,7 @@ def docrop(context: Context) -> None: @Process("sharpen") -def sharpen(context: Context) -> Optional[NpNdarrayInt]: +def sharpen(context: process_utils.Context) -> Optional[NpNdarrayInt]: """Sharpen an image.""" if ( @@ -809,14 +491,14 @@ def sharpen(context: Context) -> Optional[NpNdarrayInt]: ): return None if context.image is None: - raise ScanToPaperlessException("The image is required") + raise scan_to_paperless.ScanToPaperlessException("The image is required") image = cv2.GaussianBlur(context.image, (0, 0), 3) return cast(NpNdarrayInt, cv2.addWeighted(context.image, 1.5, image, -0.5, 0)) @Process("dither") @external -def dither(context: Context, source: str, destination: str) -> None: +def dither(context: process_utils.Context, source: str, destination: str) -> None: """Dither an image.""" if ( @@ -830,7 +512,7 @@ def dither(context: Context, source: str, destination: str) -> None: @Process("autorotate", True) -def autorotate(context: Context) -> None: +def autorotate(context: process_utils.Context) -> None: """ Auto rotate an image. @@ -989,7 +671,10 @@ def find_limit_contour( def find_limits( - image: NpNdarrayInt, vertical: bool, context: Context, contours: list[tuple[int, int, int, int]] + image: NpNdarrayInt, + vertical: bool, + context: process_utils.Context, + contours: list[tuple[int, int, int, int]], ) -> tuple[list[int], list[tuple[int, int, int, int]]]: """Find the limit for assisted split.""" contours_limits = find_limit_contour(image, vertical, contours) @@ -1035,7 +720,7 @@ def fill_limits( def find_contours( image: NpNdarrayInt, - context: Context, + context: process_utils.Context, name: str, config: schema.Contour, ) -> list[tuple[int, int, int, int]]: @@ -1052,12 +737,13 @@ def find_contours( thresh = cv2.adaptiveThreshold( gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV, block_size + 1, threshold_value_c ) - if context.is_progress() or _is_ipython(): - if _is_ipython(): + if context.is_progress() or jupyter_utils.is_ipython(): + if jupyter_utils.is_ipython(): print("Threshold") thresh_rgb = cv2.cvtColor(thresh, cv2.COLOR_GRAY2RGB) context.save_progress_images( - "threshold", thresh_rgb[context.get_index(thresh_rgb)] if _is_ipython() else thresh + "threshold", + thresh_rgb[context.get_index(thresh_rgb)] if jupyter_utils.is_ipython() else thresh, ) return _find_contours_thresh(image, thresh, context, name, config) @@ -1066,7 +752,7 @@ def find_contours( def _find_contours_thresh( image: NpNdarrayInt, thresh: NpNdarrayInt, - context: Context, + context: process_utils.Context, name: str, config: schema.Contour, ) -> list[tuple[int, int, int, int]]: @@ -1088,7 +774,9 @@ def _find_contours_thresh( for cnt in contours: x, y, width, height = cv2.boundingRect(cnt) if width > min_size and height > min_size: - contour_image = crop_image(image, x, y, width, height, context.get_background_color()) + contour_image = process_utils.crop_image( + image, x, y, width, height, context.get_background_color() + ) imagergb = ( rgba2rgb(contour_image) if len(contour_image.shape) == 3 and contour_image.shape[2] == 4 @@ -1435,379 +1123,6 @@ def _update_config(config: schema.Configuration) -> None: del old_config["args"]["rule"]["enable"] -def _pretty_repr(value: Any, prefix: str = "") -> str: - if isinstance(value, dict): - return "\n".join( - [ - "{", - *[ - f'{prefix} "{key}": {_pretty_repr(value, prefix + " ")},' - for key, value in value.items() - ], - prefix + "}", - ] - ) - - return repr(value) - - -def _is_ipython() -> bool: - try: - __IPYTHON__ # type: ignore[name-defined] # pylint: disable=pointless-statement - return True - except NameError: - return False - - -def _create_jupyter_notebook(root_folder: str, context: Context, step: schema.Step) -> None: - # Jupyter notebook - dest_folder = os.path.join(root_folder, "jupyter") - if not os.path.exists(dest_folder): - os.makedirs(dest_folder) - with open(os.path.join(dest_folder, "README.txt"), "w", encoding="utf-8") as readme_file: - readme_file.write( - """# Jupyter notebook - -Install dependencies: -pip install scan-to-paperless[process] jupyterlab Pillow - -Run: -jupyter lab - -Open the notebook file. -""" - ) - - notebook = nbformat.v4.new_notebook() # type: ignore[no-untyped-call] - - notebook["cells"].append( - nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] - """# Scan to Paperless - -This notebook show the transformation applied on the images of the document. - -At the start of each step, se set some values on the `context.config["args"]` dict, -you can change the values to see the impact on the result, -then yon can all those changes in the `config.yaml` file, in the `args` section.""" - ) - ) - - notebook["cells"].append( - nbformat.v4.new_markdown_cell("Do the required imports.") # type: ignore[no-untyped-call] - ) - notebook["cells"].append( - nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] - """import os -import cv2 -import numpy as np - -from scan_to_paperless import process""" - ) - ) - - notebook["cells"].append( - nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] - """Calculate the base folder of the document.""" - ) - ) - notebook["cells"].append( - nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] - """import IPython - -base_folder = os.path.dirname(os.path.dirname(IPython.extract_module_locals()[1]['__vsc_ipynb_file__']))""" - ) - ) - - notebook["cells"].append( - nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] - """Open on of the source images, you can change it by uncommenting the corresponding line.""" - ) - ) - other_images_open = "\n".join( - [ - f'# context.image = cv2.imread(os.path.join(base_folder, "{image}"))' - for image in step["sources"][1:] - ] - ) - notebook["cells"].append( - nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] - f"""# Open Source image -context = process.Context({{"args": {{}}}}, {{}}) - -# Open one of the images -context.image = cv2.imread(os.path.join(base_folder, "{step["sources"][0]}")) -{other_images_open} - -images_context = {{"original": context.image}}""" - ) - ) - - notebook["cells"].append( - nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] - """Set the values that's used by more than one step.""" - ) - ) - notebook["cells"].append( - nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] - f"""context.config["args"] = {{ - "dpi": {context.config["args"].get("dpi", schema.DPI_DEFAULT)}, - "background_color": {context.config["args"].get("background_color", schema.BACKGROUND_COLOR_DEFAULT)}, -}}""" - ) - ) - - notebook["cells"].append( - nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] - """Get the index that represent the part of the image we want to see.""" - ) - ) - notebook["cells"].append( - nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] - """ -# Get a part of the image to display, by default, the top of the image -context.get_index = lambda image: np.ix_( - np.arange(0, min(image.shape[1], 500)), - np.arange(0, image.shape[1]), - np.arange(0, image.shape[2]), -) - -context.display_image(images_context["original"])""" - ) - ) - - notebook["cells"].append( - nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] - """Calculate the image mask, the mask is used to hide some part of the image when we calculate the image skew and the image auto crop (based on the content). - -The `lower_hsv_color`and the `upper_hsv_color` are used to define the color range to remove, -the `de_noise_size` is used to remove noise from the image, -the `buffer_size` is used to add a buffer around the image and -the `buffer_level` is used to define the level of the buffer (`0.0` to `1.0`). - -To remove the gray background from the scanner, on document I use the following values: -```yaml -lower_hsv_color: [0, 0, 250] -upper_hsv_color: [255, 10, 255] -``` -On leaflet I use the following values: -```yaml -lower_hsv_color: [0, 20, 0] -upper_hsv_color: [255, 255, 255] -``` -""" - ) - ) - notebook["cells"].append( - nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] - f"""context.image = images_context["original"].copy() - -context.config["args"]["auto_mask"] = {_pretty_repr(context.config["args"].get("auto_mask", {}))} - -hsv = cv2.cvtColor(context.image, cv2.COLOR_BGR2HSV) -print("Hue (h)") -context.display_image(cv2.cvtColor(hsv[:, :, 0], cv2.COLOR_GRAY2RGB)) -print("Saturation (s)") -context.display_image(cv2.cvtColor(hsv[:, :, 1], cv2.COLOR_GRAY2RGB)) -print("Value (v)") -context.display_image(cv2.cvtColor(hsv[:, :, 2], cv2.COLOR_GRAY2RGB)) - -# Print the HSV value on some point of the image -points = [ - [10, 10], - [100, 100], -] -image = context.image.copy() -for x, y in points: - print(f"Pixel: {{x}}:{{y}}, with value: {{hsv[y, x, :]}}") - cv2.drawMarker(image, [x, y], (0, 0, 255), cv2.MARKER_CROSS, 20, 2) -context.display_image(image) - -context.init_mask() -if context.mask is not None: - context.display_image(cv2.cvtColor(context.mask, cv2.COLOR_GRAY2RGB)) -context.display_image(context.get_masked()) - -images_context["auto_mask"] = context.image""" - ) - ) - - notebook["cells"].append( - nbformat.v4.new_markdown_cell("Display the image histogram.") # type: ignore[no-untyped-call] - ) - notebook["cells"].append( - nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] - f"""context.image = images_context["auto_mask"].copy() - -context.config["args"]["level"] = {context.config["args"].get("level", schema.LEVEL_DEFAULT)} -context.config["args"]["cut_white"] = {context.config["args"].get("cut_white", schema.CUT_WHITE_DEFAULT)} -context.config["args"]["cut_black"] = {context.config["args"].get("cut_black", schema.CUT_BLACK_DEFAULT)} - -process.histogram(context)""" - ) - ) - - notebook["cells"].append( - nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] - """Do the image level correction. - -Some of the used values are displayed in the histogram chart.""" - ) - ) - notebook["cells"].append( - nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] - f"""context.image = images_context["auto_mask"].copy() - -context.config["args"]["auto_level"] = {context.config["args"].get("auto_level", schema.AUTO_LEVEL_DEFAULT)}, -context.config["args"]["level"] = {context.config["args"].get("level", schema.LEVEL_DEFAULT)}, - -process.level(context) -context.display_image(context.image) - -images_context["level"] = context.image""" - ) - ) - - notebook["cells"].append( - nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] - """Do the image level cut correction. - -Some of the used values are displayed in the histogram chart.""" - ) - ) - notebook["cells"].append( - nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] - f"""context.image = images_context["level"].copy() - -print(f"Use cut_white: {context.config["args"]["cut_white"]}") -print(f"Use cut_black: {context.config["args"]["cut_black"]}") - -process.color_cut(context) -context.display_image(context.image) - -images_context["color_cut"] = context.image""" - ) - ) - - notebook["cells"].append( - nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] - """Cut some part of the image by auto removing a part of the image. - -The needed of this step is to remove some part of the image that represent the part that is out of the page, witch is gray with some scanner. - -The `lower_hsv_color`and the `upper_hsv_color` are used to define the color range to remove, -the `de_noise_size` is used to remove noise from the image, -the `buffer_size` is used to add a buffer around the image and -the `buffer_level` is used to define the level of the buffer (`0.0` to `1.0`).""" - ) - ) - notebook["cells"].append( - nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] - f"""context.image = images_context["color_cut"].copy() - -context.config["args"]["auto_cut"] = {_pretty_repr(context.config["args"].get("auto_cut", {}))} - -# Print in HSV some point of the image -hsv = cv2.cvtColor(context.image, cv2.COLOR_BGR2HSV) -print("Pixel 10:10: ", hsv[10, 10]) -print("Pixel 100:100: ", hsv[100, 100]) - -process.cut(context) -context.display_image(context.image) - -images_context["cut"] = context.image""" - ) - ) - - notebook["cells"].append( - nbformat.v4.new_markdown_cell("Do the image skew correction.") # type: ignore[no-untyped-call] - ) - notebook["cells"].append( - nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] - f"""context.image = images_context["cut"].copy() - -context.config["args"]["deskew"] = {_pretty_repr(context.config["args"].get("deskew", {}))} - -# The angle can be forced in config.images_config..angle. -process.deskew(context) -context.display_image(context.image) - -images_context["deskew"] = context.image""" - ) - ) - - notebook["cells"].append( - nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] - """Do the image auto crop base on the image content.""" - ) - ) - notebook["cells"].append( - nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] - f"""context.image = images_context["deskew"].copy() - -context.config["args"]["crop"] = {_pretty_repr(context.config["args"].get("crop", {}))} - -process.docrop(context) -context.display_image(context.image) - -images_context["crop"] = context.image""" - ) - ) - - notebook["cells"].append( - nbformat.v4.new_markdown_cell("Do the image sharpen correction.") # type: ignore[no-untyped-call] - ) - notebook["cells"].append( - nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] - f"""context.image = images_context["crop"].copy() - -context.config["args"]["sharpen"] = {context.config["args"].get("sharpen", schema.SHARPEN_DEFAULT)} - -process.sharpen(context) -context.display_image(context.image) - -images_context["sharpen"] = context.image""" - ) - ) - - notebook["cells"].append( - nbformat.v4.new_markdown_cell("Do the image dither correction.") # type: ignore[no-untyped-call] - ) - notebook["cells"].append( - nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] - f"""context.image = images_context["sharpen"].copy() - -context.config["args"]["dither"] = {context.config["args"].get("dither", schema.DITHER_DEFAULT)} - -process.dither(context) -context.display_image(context.image) - -images_context["dither"] = context.image""" - ) - ) - - notebook["cells"].append( - nbformat.v4.new_markdown_cell( # type: ignore[no-untyped-call] - """Do the image auto rotate correction, based on the text orientation. - -This require Tesseract to be installed.""" - ) - ) - notebook["cells"].append( - nbformat.v4.new_code_cell( # type: ignore[no-untyped-call] - """context.image = images_context["dither"].copy() - -try: - process.autorotate(context) - context.display_image(context.image) -except FileNotFoundError as e: - print("Tesseract not found, skipping autorotate: ", e)""" - ) - ) - - with open(os.path.join(dest_folder, "jupyter.ipynb"), "w", encoding="utf-8") as jupyter_file: - nbformat.write(notebook, jupyter_file) # type: ignore[no-untyped-call] - - def transform( config: schema.Configuration, step: schema.Step, @@ -1830,9 +1145,9 @@ def transform( if status is not None: status.set_status(config_file_name, -1, f"Transform ({os.path.basename(image)})", write=True) image_name = f"{os.path.basename(image).rsplit('.')[0]}.png" - context = Context(config, step, config_file_name, root_folder, image_name) + context = process_utils.Context(config, step, config_file_name, root_folder, image_name) if context.image_name is None: - raise ScanToPaperlessException("Image name is required") + raise scan_to_paperless.ScanToPaperlessException("Image name is required") context.image = cv2.imread(os.path.join(root_folder, image)) assert context.image is not None images_config = context.config.setdefault("images_config", {}) @@ -2013,7 +1328,9 @@ def transform( images.append(img2) process_count = context.process_count - _create_jupyter_notebook(root_folder, context, step) + from scan_to_paperless import jupyter # pylint: disable=import-outside-toplevel + + jupyter.create_transform_notebook(root_folder, context, step) progress = os.environ.get("PROGRESS", "FALSE") == "TRUE" @@ -2123,7 +1440,9 @@ def _save_progress(root_folder: Optional[str], count: int, name: str, image_name print(exception) -def save(context: Context, root_folder: str, image: str, folder: str, force: bool = False) -> str: +def save( + context: process_utils.Context, root_folder: str, image: str, folder: str, force: bool = False +) -> str: """Save the current image in a subfolder if progress mode in enabled.""" if force or context.is_progress(): @@ -2167,7 +1486,7 @@ def split( nb_horizontal += 1 if nb_vertical * nb_horizontal != len(assisted_split["destinations"]): - raise ScanToPaperlessException( + raise scan_to_paperless.ScanToPaperlessException( f"Wrong number of destinations ({len(assisted_split['destinations'])}), " f"vertical: {nb_horizontal}, height: {nb_vertical}, image: '{assisted_split['source']}'" ) @@ -2182,7 +1501,7 @@ def split( transformed_images = [] for assisted_split in config["assisted_split"]: image = assisted_split["source"] - context = Context(config, step) + context = process_utils.Context(config, step) width, height = ( int(e) for e in output(CONVERT + [image, "-format", "%w %h", "info:-"]).strip().split(" ") ) @@ -2266,7 +1585,7 @@ def split( items: list[Item] = append[page_number] vertical = len(horizontal_limits) == 0 if not vertical and len(vertical_limits) != 0 and len(items) > 1: - raise ScanToPaperlessException(f"Mix of limit type for page '{page_number}'") + raise scan_to_paperless.ScanToPaperlessException(f"Mix of limit type for page '{page_number}'") with tempfile.NamedTemporaryFile(suffix=".png") as process_file: call( @@ -2454,7 +1773,7 @@ def finalize( data = {"title": title} response = requests.post(url, headers=headers, data=data, files=files, timeout=120) if not response.ok: - raise ScanToPaperlessException( + raise scan_to_paperless.ScanToPaperlessException( f"Failed ({response.status_code}) upload to " f"'{url}' with token '{token}'\n{response.text}" ) diff --git a/scan_to_paperless/process_utils.py b/scan_to_paperless/process_utils.py new file mode 100644 index 00000000..22cd4ec7 --- /dev/null +++ b/scan_to_paperless/process_utils.py @@ -0,0 +1,335 @@ +"""Utility functions and context used in the process.""" + +import logging +import math +import os +from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast + +import cv2 +import numpy as np +from PIL import Image + +import scan_to_paperless +import scan_to_paperless.jupyter_utils +import scan_to_paperless.status +from scan_to_paperless import process_schema as schema + +if TYPE_CHECKING: + NpNdarrayInt = np.ndarray[np.uint8, Any] +else: + NpNdarrayInt = np.ndarray + +_LOG = logging.getLogger(__name__) + + +def rotate_image( + image: NpNdarrayInt, angle: float, background: Union[int, tuple[int, int, int]] +) -> NpNdarrayInt: + """Rotate the image.""" + + old_width, old_height = image.shape[:2] + angle_radian = math.radians(angle) + width = abs(np.sin(angle_radian) * old_height) + abs(np.cos(angle_radian) * old_width) + height = abs(np.sin(angle_radian) * old_width) + abs(np.cos(angle_radian) * old_height) + + image_center: tuple[Any, ...] = tuple(np.array(image.shape[1::-1]) / 2) + rot_mat = cv2.getRotationMatrix2D(image_center, angle, 1.0) + rot_mat[1, 2] += (width - old_width) / 2 + rot_mat[0, 2] += (height - old_height) / 2 + return cast( + NpNdarrayInt, + cv2.warpAffine(image, rot_mat, (int(round(height)), int(round(width))), borderValue=background), + ) + + +def crop_image( # pylint: disable=too-many-arguments + image: NpNdarrayInt, + x: int, + y: int, + width: int, + height: int, + background: Union[tuple[int], tuple[int, int, int]], +) -> NpNdarrayInt: + """Crop the image.""" + + matrix: NpNdarrayInt = np.array([[1.0, 0.0, -x], [0.0, 1.0, -y]]) + return cast( + NpNdarrayInt, + cv2.warpAffine(image, matrix, (int(round(width)), int(round(height))), borderValue=background), + ) + + +class Context: # pylint: disable=too-many-instance-attributes + """All the context of the current image with his mask.""" + + def __init__( # pylint: disable=too-many-arguments + self, + config: schema.Configuration, + step: schema.Step, + config_file_name: Optional[str] = None, + root_folder: Optional[str] = None, + image_name: Optional[str] = None, + ) -> None: + """Initialize.""" + + self.config = config + self.step = step + self.config_file_name = config_file_name + self.root_folder = root_folder + self.image_name = image_name + self.image: Optional[NpNdarrayInt] = None + self.mask: Optional[NpNdarrayInt] = None + self.get_index: Callable[ + [NpNdarrayInt], Optional[tuple[np.ndarray[Any, np.dtype[np.signedinteger[Any]]], ...]] + ] = lambda image: np.ix_( + np.arange(0, image.shape[1]), + np.arange(0, image.shape[1]), + np.arange(0, image.shape[2]), + ) + + self.process_count = self.step.get("process_count", 0) + + def _get_mask( + self, + auto_mask_config: Optional[schema.AutoMask], + config_section: str, + default_file_name: str, + ) -> Optional[NpNdarrayInt]: + """Init the mask.""" + + if auto_mask_config is not None: + hsv = cv2.cvtColor(self.image, cv2.COLOR_BGR2HSV) + + lower_val = np.array( + auto_mask_config.setdefault("lower_hsv_color", schema.LOWER_HSV_COLOR_DEFAULT) + ) + upper_val = np.array( + auto_mask_config.setdefault("upper_hsv_color", schema.UPPER_HSV_COLOR_DEFAULT) + ) + mask = cv2.inRange(hsv, lower_val, upper_val) + + de_noise_size = auto_mask_config.setdefault("de_noise_size", schema.DE_NOISE_SIZE_DEFAULT) + mask = cv2.copyMakeBorder( + mask, + de_noise_size, + de_noise_size, + de_noise_size, + de_noise_size, + cv2.BORDER_REPLICATE, + ) + if auto_mask_config.setdefault("de_noise_morphology", schema.DE_NOISE_MORPHOLOGY_DEFAULT): + mask = cv2.morphologyEx( + mask, + cv2.MORPH_CLOSE, + cv2.getStructuringElement(cv2.MORPH_RECT, (de_noise_size, de_noise_size)), + ) + else: + blur = cv2.blur( + mask, + (de_noise_size, de_noise_size), + ) + _, mask = cv2.threshold( + blur, + auto_mask_config.setdefault("de_noise_level", schema.DE_NOISE_LEVEL_DEFAULT), + 255, + cv2.THRESH_BINARY, + ) + + inverse_mask = auto_mask_config.setdefault("inverse_mask", schema.INVERSE_MASK_DEFAULT) + if not inverse_mask: + mask = cv2.bitwise_not(mask) + + buffer_size = auto_mask_config.setdefault("buffer_size", schema.BUFFER_SIZE_DEFAULT) + blur = cv2.blur(mask, (buffer_size, buffer_size)) + _, mask = cv2.threshold( + blur, + auto_mask_config.setdefault("buffer_level", schema.BUFFER_LEVEL_DEFAULT), + 255, + cv2.THRESH_BINARY, + ) + + mask = mask[de_noise_size:-de_noise_size, de_noise_size:-de_noise_size] + + if self.root_folder: + mask_file: str = os.path.join(self.root_folder, default_file_name) + assert mask_file + if not os.path.exists(mask_file): + base_folder = os.path.dirname(self.root_folder) + assert base_folder + mask_file = os.path.join(base_folder, default_file_name) + if not os.path.exists(mask_file): + mask_file = "" + mask_file = auto_mask_config.setdefault("additional_filename", mask_file) + if mask_file and os.path.exists(mask_file): + mask = cv2.add( + mask, + cv2.bitwise_not( + cv2.resize( + cv2.imread(mask_file, cv2.IMREAD_GRAYSCALE), + (mask.shape[1], mask.shape[0]), + ) + ), + ) + + final_mask = cv2.bitwise_not(mask) + + if os.environ.get("PROGRESS", "FALSE") == "TRUE" and self.root_folder: + self.save_progress_images(config_section.replace("_", "-"), final_mask) + elif self.root_folder: + mask_file = os.path.join(self.root_folder, default_file_name) + if not os.path.exists(mask_file): + base_folder = os.path.dirname(self.root_folder) + assert base_folder + mask_file = os.path.join(base_folder, default_file_name) + if not os.path.exists(mask_file): + return None + + final_mask = cv2.imread(mask_file, cv2.IMREAD_GRAYSCALE) + if self.image is not None and final_mask is not None: + return cast(NpNdarrayInt, cv2.resize(final_mask, (self.image.shape[1], self.image.shape[0]))) + return cast(NpNdarrayInt, final_mask) + + def init_mask(self) -> None: + """Init the mask image used to mask the image on the crop and skew calculation.""" + + auto_mask_config = self.config["args"].setdefault( + "auto_mask", cast(schema.AutoMaskOperation, schema.AUTO_MASK_OPERATION_DEFAULT) + ) + self.mask = ( + self._get_mask( + auto_mask_config.setdefault("auto_mask", {}), + "auto_mask", + "mask.png", + ) + if auto_mask_config.setdefault("enabled", schema.AUTO_MASK_ENABLED_DEFAULT) + else None + ) + + def get_background_color(self) -> tuple[int, int, int]: + """Get the background color.""" + + return cast( + tuple[int, int, int], + self.config["args"].setdefault("background_color", schema.BACKGROUND_COLOR_DEFAULT), + ) + + def do_initial_cut(self) -> None: + """Definitively mask the original image.""" + + if "auto_cut" in self.config["args"]: + assert self.image is not None + mask = self._get_mask( + self.config["args"] + .setdefault("auto_cut", cast(schema.AutoCut, schema.AUTO_CUT_DEFAULT)) + .setdefault("auto_mask", {}), + "auto_cut", + "cut.png", + ) + self.image[mask == 0] = self.get_background_color() + + def get_process_count(self) -> int: + """Get the step number.""" + + try: + return self.process_count + finally: + self.process_count += 1 + + def get_masked(self) -> NpNdarrayInt: + """Get the mask.""" + + if self.image is None: + raise scan_to_paperless.ScanToPaperlessException("The image is None") + if self.mask is None: + return self.image.copy() + + image = self.image.copy() + image[self.mask == 0] = self.get_background_color() + return image + + def crop(self, x: int, y: int, width: int, height: int) -> None: + """Crop the image.""" + + if self.image is None: + raise scan_to_paperless.ScanToPaperlessException("The image is None") + self.image = crop_image(self.image, x, y, width, height, self.get_background_color()) + if self.mask is not None: + self.mask = crop_image(self.mask, x, y, width, height, (0,)) + + def rotate(self, angle: float) -> None: + """Rotate the image.""" + + if self.image is None: + raise scan_to_paperless.ScanToPaperlessException("The image is None") + self.image = rotate_image(self.image, angle, self.get_background_color()) + if self.mask is not None: + self.mask = rotate_image(self.mask, angle, 0) + + def get_px_value(self, value: Union[int, float]) -> float: + """Get the value in px.""" + + return value / 10 / 2.51 * self.config["args"].setdefault("dpi", schema.DPI_DEFAULT) + + def is_progress(self) -> bool: + """Return we want to have the intermediate files.""" + + return os.environ.get("PROGRESS", "FALSE") == "TRUE" or self.config.setdefault( + "progress", schema.PROGRESS_DEFAULT + ) + + def save_progress_images( + self, + name: str, + image: Optional[NpNdarrayInt] = None, + image_prefix: str = "", + process_count: Optional[int] = None, + force: bool = False, + ) -> Optional[str]: + """Save the intermediate images.""" + + if scan_to_paperless.jupyter_utils.is_ipython(): + if image is None: + return None + + from IPython.display import display # pylint: disable=import-outside-toplevel + + display(Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))) # type: ignore[no-untyped-call] + return None + + if process_count is None: + process_count = self.get_process_count() + if (self.is_progress() or force) and self.image_name is not None and self.root_folder is not None: + name = f"{process_count}-{name}" if self.is_progress() else name + dest_folder = os.path.join(self.root_folder, name) + if not os.path.exists(dest_folder): + os.makedirs(dest_folder) + dest_image = os.path.join(dest_folder, image_prefix + self.image_name) + if image is not None: + try: + cv2.imwrite(dest_image, image) + return dest_image + except Exception as exception: + print(exception) + else: + try: + cv2.imwrite(dest_image, self.image) + except Exception as exception: + print(exception) + dest_image = os.path.join(dest_folder, "mask-" + self.image_name) + try: + dest_image = os.path.join(dest_folder, "masked-" + self.image_name) + except Exception as exception: + print(exception) + try: + cv2.imwrite(dest_image, self.get_masked()) + except Exception as exception: + print(exception) + return None + + def display_image(self, image: NpNdarrayInt) -> None: + """Display the image.""" + + if scan_to_paperless.jupyter_utils.is_ipython(): + from IPython.display import display # pylint: disable=import-outside-toplevel + + display(Image.fromarray(cv2.cvtColor(image[self.get_index(image)], cv2.COLOR_BGR2RGB))) # type: ignore[no-untyped-call] diff --git a/spell-ignore-words.txt b/spell-ignore-words.txt new file mode 100644 index 00000000..2f4f122c --- /dev/null +++ b/spell-ignore-words.txt @@ -0,0 +1 @@ +jupyter diff --git a/tests/test_process.py b/tests/test_process.py index 270b81b6..35f93e43 100644 --- a/tests/test_process.py +++ b/tests/test_process.py @@ -5,13 +5,15 @@ import subprocess import cv2 +import nbformat import pikepdf import pytest import skimage.color import skimage.io from c2cwsgiutils.acceptance.image import check_image, check_image_file +from nbconvert.preprocessors import ExecutePreprocessor -from scan_to_paperless import code, process +from scan_to_paperless import code, process, process_utils REGENERATE = False @@ -32,7 +34,7 @@ def test_find_lines(): # @pytest.mark.skip(reason="for test") def test_find_limit_contour(): - context = process.Context({"args": {}}, {}) + context = process_utils.Context({"args": {}}, {}) context.image = load_image("limit-contour-1.png") contours = process.find_contours(context.image, context, "limit", {}) limits = process.find_limit_contour(context.image, True, contours) @@ -47,37 +49,41 @@ def test_crop(): os.makedirs(root_folder) check_image( root_folder, - cv2.cvtColor(process.crop_image(image, 100, 0, 100, 300, (255, 255, 255)), cv2.COLOR_BGR2RGB), + cv2.cvtColor(process_utils.crop_image(image, 100, 0, 100, 300, (255, 255, 255)), cv2.COLOR_BGR2RGB), os.path.join(os.path.dirname(__file__), "crop-1.expected.png"), generate_expected_image=REGENERATE, ) check_image( root_folder, - cv2.cvtColor(process.crop_image(image, 0, 100, 300, 100, (255, 255, 255)), cv2.COLOR_BGR2RGB), + cv2.cvtColor(process_utils.crop_image(image, 0, 100, 300, 100, (255, 255, 255)), cv2.COLOR_BGR2RGB), os.path.join(os.path.dirname(__file__), "crop-2.expected.png"), generate_expected_image=REGENERATE, ) check_image( root_folder, - cv2.cvtColor(process.crop_image(image, 100, -100, 100, 200, (255, 255, 255)), cv2.COLOR_BGR2RGB), + cv2.cvtColor( + process_utils.crop_image(image, 100, -100, 100, 200, (255, 255, 255)), cv2.COLOR_BGR2RGB + ), os.path.join(os.path.dirname(__file__), "crop-3.expected.png"), generate_expected_image=REGENERATE, ) check_image( root_folder, - cv2.cvtColor(process.crop_image(image, -100, 100, 200, 100, (255, 255, 255)), cv2.COLOR_BGR2RGB), + cv2.cvtColor( + process_utils.crop_image(image, -100, 100, 200, 100, (255, 255, 255)), cv2.COLOR_BGR2RGB + ), os.path.join(os.path.dirname(__file__), "crop-4.expected.png"), generate_expected_image=REGENERATE, ) check_image( root_folder, - cv2.cvtColor(process.crop_image(image, 100, 200, 100, 200, (255, 255, 255)), cv2.COLOR_BGR2RGB), + cv2.cvtColor(process_utils.crop_image(image, 100, 200, 100, 200, (255, 255, 255)), cv2.COLOR_BGR2RGB), os.path.join(os.path.dirname(__file__), "crop-5.expected.png"), generate_expected_image=REGENERATE, ) check_image( root_folder, - cv2.cvtColor(process.crop_image(image, 200, 100, 200, 100, (255, 255, 255)), cv2.COLOR_BGR2RGB), + cv2.cvtColor(process_utils.crop_image(image, 200, 100, 200, 100, (255, 255, 255)), cv2.COLOR_BGR2RGB), os.path.join(os.path.dirname(__file__), "crop-6.expected.png"), generate_expected_image=REGENERATE, ) @@ -90,40 +96,40 @@ def test_rotate(): root_folder = "/results/rotate" if not os.path.exists(root_folder): os.makedirs(root_folder) - image = process.crop_image(image, 0, 50, 300, 200, (255, 255, 255)) + image = process_utils.crop_image(image, 0, 50, 300, 200, (255, 255, 255)) check_image( root_folder, - cv2.cvtColor(process.rotate_image(image, 10, (255, 255, 255)), cv2.COLOR_BGR2RGB), + cv2.cvtColor(process_utils.rotate_image(image, 10, (255, 255, 255)), cv2.COLOR_BGR2RGB), os.path.join(os.path.dirname(__file__), "rotate-1.expected.png"), generate_expected_image=REGENERATE, ) check_image( root_folder, - cv2.cvtColor(process.rotate_image(image, -10, (255, 255, 255)), cv2.COLOR_BGR2RGB), + cv2.cvtColor(process_utils.rotate_image(image, -10, (255, 255, 255)), cv2.COLOR_BGR2RGB), os.path.join(os.path.dirname(__file__), "rotate-2.expected.png"), generate_expected_image=REGENERATE, ) check_image( root_folder, - cv2.cvtColor(process.rotate_image(image, 90, (255, 255, 255)), cv2.COLOR_BGR2RGB), + cv2.cvtColor(process_utils.rotate_image(image, 90, (255, 255, 255)), cv2.COLOR_BGR2RGB), os.path.join(os.path.dirname(__file__), "rotate-3.expected.png"), generate_expected_image=REGENERATE, ) check_image( root_folder, - cv2.cvtColor(process.rotate_image(image, -90, (255, 255, 255)), cv2.COLOR_BGR2RGB), + cv2.cvtColor(process_utils.rotate_image(image, -90, (255, 255, 255)), cv2.COLOR_BGR2RGB), os.path.join(os.path.dirname(__file__), "rotate-4.expected.png"), generate_expected_image=REGENERATE, ) check_image( root_folder, - cv2.cvtColor(process.rotate_image(image, 270, (255, 255, 255)), cv2.COLOR_BGR2RGB), + cv2.cvtColor(process_utils.rotate_image(image, 270, (255, 255, 255)), cv2.COLOR_BGR2RGB), os.path.join(os.path.dirname(__file__), "rotate-4.expected.png"), generate_expected_image=REGENERATE, ) check_image( root_folder, - cv2.cvtColor(process.rotate_image(image, 180, (255, 255, 255)), cv2.COLOR_BGR2RGB), + cv2.cvtColor(process_utils.rotate_image(image, 180, (255, 255, 255)), cv2.COLOR_BGR2RGB), os.path.join(os.path.dirname(__file__), "rotate-5.expected.png"), generate_expected_image=REGENERATE, ) @@ -697,7 +703,7 @@ def test_multi_code(): # @pytest.mark.skip(reason="for test") -def test_tiff(): +def test_tiff_jupyter(): init_test() root_folder = "/results/tiff" source_folder = os.path.join(root_folder, "source") @@ -719,7 +725,12 @@ def test_tiff(): config_file_name = os.path.join(root_folder, "config.yaml") step = process.transform(config, step, config_file_name, root_folder) assert step["sources"] == ["/results/tiff/image-1.png"] - assert list(glob.glob(f"{root_folder}/**/*.tiff")) == ["/results/tiff/source/image-1.tiff"] + assert list(glob.glob(f"{root_folder}/**/*.tiff")) == [os.path.join(root_folder, "source/image-1.tiff")] + + with open("/results/tiff/jupyter/jupyter.ipynb") as f: + nb = nbformat.read(f, as_version=4) + ep = ExecutePreprocessor(timeout=600, kernel_name="python3") + ep.preprocess(nb, {"metadata": {"path": "/results/tiff/jupyter/"}}) # @pytest.mark.skip(reason="for test") @@ -756,7 +767,7 @@ def test_tiff(): ) def test_auto_mask(config, name): init_test() - context = process.Context({"args": {"auto_mask": {"auto_mask": config}}}, {}) + context = process_utils.Context({"args": {"auto_mask": {"auto_mask": config}}}, {}) context.image = cv2.imread(os.path.join(os.path.dirname(__file__), "auto-mask-source.png")) context.init_mask() check_image( @@ -770,7 +781,7 @@ def test_auto_mask(config, name): # @pytest.mark.skip(reason="for test") def test_auto_mask_combine(): init_test() - context = process.Context({"args": {"auto_mask": {}}}, {}) + context = process_utils.Context({"args": {"auto_mask": {}}}, {}) context.image = cv2.imread(os.path.join(os.path.dirname(__file__), "auto-mask-source.png")) context.root_folder = os.path.join(os.path.join(os.path.dirname(__file__), "auto-mask-other")) context.image_name = "image.png" @@ -786,7 +797,7 @@ def test_auto_mask_combine(): # @pytest.mark.skip(reason="for test") def test_auto_cut(): init_test() - context = process.Context({"args": {"auto_cut": {}, "background_color": [255, 0, 0]}}, {}) + context = process_utils.Context({"args": {"auto_cut": {}, "background_color": [255, 0, 0]}}, {}) context.image = cv2.imread(os.path.join(os.path.dirname(__file__), "auto-mask-source.png")) context.do_initial_cut() check_image( @@ -800,7 +811,7 @@ def test_auto_cut(): # @pytest.mark.skip(reason="for test") def test_color_cut(): init_test() - context = process.Context({"args": {"cut_white": 200}}, {}) + context = process_utils.Context({"args": {"cut_white": 200}}, {}) context.image = cv2.imread(os.path.join(os.path.dirname(__file__), "white-cut.png")) process.color_cut(context) check_image( @@ -814,7 +825,7 @@ def test_color_cut(): # @pytest.mark.skip(reason="for test") def test_histogram(): init_test() - context = process.Context( + context = process_utils.Context( {"args": {"level": True, "min_level": 10, "max_level": 90, "cut_black": 20, "cut_white": 200}}, {} ) context.image = cv2.imread(os.path.join(os.path.dirname(__file__), "limit-contour-all-1.png"))