Merge pull request #213 from NREL/gtf-remote-libraries

Remote libraries (and bulk library loading)

.gitignore

@@ -24,7 +24,7 @@ docs/reference/apidoc/_autosummary
 
 # Jupyter Notebook
 *.ipynb_checkpoints
-
+notebooks/output.ttl # generated by 223PExample
 
 
 # Environments

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
     -   id: black
         entry: black --check
 -   repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
     -    id: isort
          entry: isort --check

buildingmotif/bin/cli.py

@@ -0,0 +1,168 @@
+import argparse
+import logging
+import shutil
+import sys
+from os import getenv
+from pathlib import Path
+
+from buildingmotif import BuildingMOTIF
+from buildingmotif.dataclasses import Library
+
+cli = argparse.ArgumentParser(
+    prog="buildingmotif", description="CLI Interface for common BuildingMOTIF tasks"
+)
+subparsers = cli.add_subparsers(dest="subcommand")
+subcommands = {}
+log = logging.getLogger()
+log.setLevel(logging.INFO)
+
+ONTOLOGY_FILE_SUFFIXES = ["ttl", "n3", "ntriples", "xml"]
+
+
+# borrowing some ideas from https://gist.github.com/mivade/384c2c41c3a29c637cb6c603d4197f9f
+def arg(*argnames, **kwargs):
+    """Helper for defining arguments on subcommands"""
+    return argnames, kwargs
+
+
+def subcommand(*subparser_args, parent=subparsers):
+    """Decorates a function and makes it available as a subcommand"""
+
+    def decorator(func):
+        parser = parent.add_parser(func.__name__, description=func.__doc__)
+        for args, kwargs in subparser_args:
+            parser.add_argument(*args, **kwargs)
+        parser.set_defaults(func=func)
+        subcommands[func] = parser
+
+    return decorator
+
+
+def get_db_uri(args) -> str:
+    """
+    Fetches the db uri from args, or prints the usage
+    for the corresponding subcommand
+    """
+    db_uri = args.db
+    if db_uri is not None:
+        return db_uri
+    db_uri = getenv("DB_URI")
+    if db_uri is not None:
+        return db_uri
+    try:
+        import configs as building_motif_configs
+    except ImportError:
+        print("No DB URI could be found")
+        print("No configs.py file found")
+        subcommands[args.func].print_help()
+        sys.exit(1)
+    db_uri = building_motif_configs.DB_URI
+    if db_uri is not None:
+        return db_uri
+    print("No DB URI could be found")
+    subcommands[args.func].print_help()
+    sys.exit(1)
+
+
+@subcommand(
+    arg(
+        "-d",
+        "--db",
+        help="Database URI of the BuildingMOTIF installation. "
+        'Defaults to $DB_URI and then contents of "config.py"',
+    ),
+    arg(
+        "--dir",
+        help="Path to a local directory containing the library",
+        nargs="+",
+    ),
+    arg(
+        "-o",
+        "--ont",
+        help="Remote URL or local file path to an RDF ontology",
+        nargs="+",
+    ),
+    arg(
+        "-l",
+        "--libraries",
+        help="Filename of the libraries YAML file specifying what "
+        "should be loaded into BuildingMOTIF",
+        default="libraries.yml",
+        nargs="+",
+        dest="library_manifest_file",
+    ),
+)
+def load(args):
+    """
+    Loads libraries from (1) local directories (--dir),
+    (2) local or remote ontology files (--ont)
+    (3) library spec file (--libraries): the provided YML file into the
+      BuildingMOTIF instance at $DB_URI or whatever is in 'configs.py'.
+      Use 'get_default_libraries_yml' for the format of the expected libraries.yml file
+    """
+    db_uri = get_db_uri(args)
+    bm = BuildingMOTIF(db_uri)
+    bm.setup_tables()
+    for directory in args.dir or []:
+        Library.load(directory=directory)
+    for ont in args.ont or []:
+        Library.load(ontology_graph=ont)
+    for library_manifest_file in args.library_manifest_file or []:
+        manifest_path = Path(library_manifest_file)
+        log.info(f"Loading buildingmotif libraries listed in {manifest_path}")
+        Library.load_from_libraries_yml(str(manifest_path))
+    bm.session.commit()
+
+
+@subcommand()
+def get_default_libraries_yml(_args):
+    """
+    Creates a default 'libraries.default.yml' file in the current directory
+    that can be edited and used with buildingmotif
+    """
+    default_file = (
+        Path(__file__).resolve().parents[1] / "resources" / "libraries.default.yml"
+    )
+    shutil.copyfile(default_file, "libraries.default.yml")
+    print("libraries.default.yml created in the current directory")
+
+
+@subcommand(
+    arg(
+        "-b",
+        "--bind",
+        help="Address on which to bind the API server",
+        default="localhost",
+    ),
+    arg(
+        "-p", "--port", help="Listening port for the API server", type=int, default=5000
+    ),
+    arg(
+        "-d",
+        "--db",
+        help="Database URI of the BuildingMOTIF installation. "
+        'Defaults to $DB_URI and then contents of "config.py"',
+    ),
+)
+def serve(args):
+    """
+    Serves the BuildingMOTIF API on the indicated host:port
+    """
+    from buildingmotif.api.app import create_app
+
+    db_uri = get_db_uri(args)
+    webapp = create_app(db_uri)
+    webapp.run(host=args.host, port=args.port, threaded=False)
+
+
+def app():
+    args = cli.parse_args()
+    if args.subcommand is None:
+        cli.print_help()
+    else:
+        args.func(args)
+
+
+# entrypoint is actually defined in pyproject.toml; this is here for convenience/testing
+if __name__ == "__main__":
+    app()

buildingmotif/dataclasses/library.py

@@ -1,8 +1,10 @@
 import logging
 import pathlib
+import tempfile
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Union
 
+import pygit2
 import pyshacl
 import rdflib
 import sqlalchemy
@@ -17,6 +19,7 @@
 from buildingmotif.dataclasses.shape_collection import ShapeCollection
 from buildingmotif.dataclasses.template import Template
 from buildingmotif.namespaces import XSD
+from buildingmotif.schemas import validate_libraries_yaml
 from buildingmotif.template_compilation import compile_template_spec
 from buildingmotif.utils import get_ontology_files, get_template_parts_from_shape
 
@@ -144,8 +147,8 @@ def load(
         :param db_id: the unique id of the library in the database,
             defaults to None
         :type db_id: Optional[int], optional
-        :param ontology_graph: a path to a serialized RDF graph,
-            defaults to None
+        :param ontology_graph: a path to a serialized RDF graph.
+            Supports remote ontology URLs, defaults to None
         :type ontology_graph: Optional[str|rdflib.Graph], optional
         :param directory: a path to a directory containing a library,
             or an rdflib graph, defaults to None
@@ -342,6 +345,24 @@ def _load_from_directory(
 
         return lib
 
+    @classmethod
+    def load_from_libraries_yml(cls, filename: str):
+        """
+        Loads *multiple* libraries from a properly-formatted 'libraries.yml'
+        file. Does not return a Library! You will need to load the libraries by
+        name in order to get the dataclasses.Library object. We recommend loading
+        libraries directly, one-by-one, in most cases. This method is here to support
+        the commandline tool.
+
+        :param filename: the filename of the YAML file to load library names from
+        :type filename: str
+        :rtype: None
+        """
+        libraries = yaml.load(open(filename, "r"), Loader=yaml.FullLoader)
+        validate_libraries_yaml(libraries)  # raises exception
+        for description in libraries:
+            _resolve_library_definition(description)
+
     @staticmethod
     def _library_exists(library_name: str) -> bool:
         """Checks whether a library with the given name exists in the database."""
@@ -498,3 +519,35 @@ def get_template_by_name(self, name: str) -> Template:
         if dbt.library_id != self._id:
             raise ValueError(f"Template {name} not in library {self._name}")
         return Template.load(dbt.id)
+
+
+def _resolve_library_definition(desc: Dict[str, Any]):
+    """
+    Loads a library from a description in libraries.yml
+    """
+    if "directory" in desc:
+        spath = pathlib.Path(desc["directory"]).absolute()
+        if spath.exists() and spath.is_dir():
+            logging.info(f"Load local library {spath} (directory)")
+            Library.load(directory=str(spath))
+        else:
+            raise Exception(f"{spath} is not an existing directory")
+    elif "ontology" in desc:
+        ont = desc["ontology"]
+        g = rdflib.Graph().parse(ont, format=rdflib.util.guess_format(ont))
+        logging.info(f"Load library {ont} as ontology graph")
+        Library.load(ontology_graph=g)
+    elif "git" in desc:
+        repo = desc["git"]["repo"]
+        branch = desc["git"]["branch"]
+        path = desc["git"]["path"]
+        logging.info(f"Load library {path} from git repository: {repo}@{branch}")
+        with tempfile.TemporaryDirectory() as temp_loc:
+            pygit2.clone_repository(
+                repo, temp_loc, checkout_branch=branch
+            )  # , depth=1)
+            new_path = pathlib.Path(temp_loc) / pathlib.Path(path)
+            if new_path.is_dir():
+                _resolve_library_definition({"directory": new_path})
+            else:
+                _resolve_library_definition({"ontology": new_path})

buildingmotif/resources/libraries.default.yml

@@ -0,0 +1,10 @@
+# load a library from a local directory
+- directory: libraries/ZonePAC/
+# load an ontology library from a remote URL
+- ontology: https://github.com/BrickSchema/Brick/releases/download/nightly/Brick.ttl
+# load a directory from a remote directory hosted in a github repo
+- git:
+    repo: https://github.com/NREL/BuildingMOTIF
+    branch: main
+    path: libraries/chiller-plant
+

buildingmotif/schemas.py

@@ -0,0 +1,47 @@
+from typing import Any
+
+import jsonschema
+
+LIBRARIES_YAML_SCHEMA = {
+    "type": "array",
+    "items": {
+        "type": "object",
+        "oneOf": [
+            {
+                "type": "object",
+                "properties": {"directory": {"type": "string"}},
+                "required": ["directory"],
+            },
+            {
+                "type": "object",
+                "properties": {"ontology": {"type": "string"}},
+                "required": ["ontology"],
+            },
+            {
+                "type": "object",
+                "properties": {
+                    "git": {
+                        "type": "object",
+                        "properties": {
+                            "repo": {"type": "string"},
+                            "branch": {"type": "string"},
+                            "path": {"type": "string"},
+                        },
+                        "required": ["repo", "branch", "path"],
+                    }
+                },
+                "required": ["git"],
+            },
+        ],
+    },
+}
+
+
+def validate_libraries_yaml(doc: Any):
+    """
+    Validates a given document against the library.yml schema. Raises
+    a jsonschema.exceptions.ValidationError if errors are found
+
+    :param doc: a value retrieved from deserializing libraries.yml file
+    """
+    jsonschema.validate(schema=LIBRARIES_YAML_SCHEMA, instance=doc)

docs/_toc.yml

@@ -8,6 +8,7 @@ parts:
 - caption: Reference
   chapters:
   - file: reference/developer_documentation.md
+  - file: reference/cli_tool.md
   - file: reference/apidoc/index.rst
 - caption: Tutorials
   chapters: