Merge pull request #36 from xchem/uploader

Adds data uploader module

Author: tdudgeon

.gitignore

@@ -9,5 +9,13 @@ venv/
 xchemalign/xchem_align.egg-info/
 xchemalign/_version.py
 test-data/outputs/upload_*
+test-data/outputs/upload-v2/upload_*
+test-data/outputs/upload-v2/*.log
+test-data/outputs/upload-v2/config.yaml
+
 example-simple
 build
+
+*~
+\#*#
+\#*

pyproject.toml

@@ -22,7 +22,10 @@ dependencies = [
     'GitPython == 3.1.41',
     'ligand_neighbourhood_alignment@git+https://github.com/xchem/ligand_neighbourhood_alignment',
     'pytest',
-    'pytest-order'
+    'pytest-order',
+    'tqdm',
+    'requests',
+    'requests-toolbelt'
 ] # Add project dependencies here, e.g. ["click", "numpy"]
 dynamic = ["version"]
 license.file = "LICENSE"

scripts/align.py

@@ -11,28 +11,17 @@
 def main():
     parser = argparse.ArgumentParser(description="aligner")
 
-    parser.add_argument("-d", "--version-dir", required=True, help="Path to version dir")
-    parser.add_argument(
-        "-m", "--metadata_file", default=Constants.METADATA_XTAL_FILENAME.format(""), help="Metadata YAML file"
-    )
-    parser.add_argument("-a", "--assemblies", help="Assemblies YAML file")
+    parser.add_argument("-d", "--dir", help="Working directory")
 
-    parser.add_argument("-l", "--log-file", help="File to write logs to")
     parser.add_argument("--log-level", type=int, default=0, help="Logging level")
     parser.add_argument("--validate", action="store_true", help="Only perform validation")
 
     args = parser.parse_args()
 
-    if args.log_file:
-        log = args.log_file
-    else:
-        log = str(Path(args.version_dir).parent / 'aligner.log')
-        print("Using {} for log file".format(str(log)))
+    print("aligner: ", args)
 
-    logger = utils.Logger(logfile=log, level=args.log_level)
-    logger.info("aligner: ", args)
-
-    a = Aligner(args.version_dir, args.metadata_file, args.assemblies, logger=logger)
+    a = Aligner(args.dir, log_level=args.log_level)
+    logger = a.logger
     num_errors, num_warnings = a.validate()
 
     if not args.validate:

scripts/collate.py

@@ -8,16 +8,16 @@
 def main():
     parser = argparse.ArgumentParser(description="collator")
 
-    parser.add_argument("-c", "--config-file", default="config.yaml", help="Configuration file")
-    parser.add_argument("-l", "--log-file", help="File to write logs to")
+    parser.add_argument("-d", "--dir", help="Working directory")
     parser.add_argument("--log-level", type=int, default=0, help="Logging level")
     parser.add_argument("-v", "--validate", action="store_true", help="Only perform validation")
 
     args = parser.parse_args()
 
-    c = Collator(args.config_file, log_file=args.log_file, log_level=args.log_level)
+    print("collator: ", str(args))
+
+    c = Collator(args.dir, log_level=args.log_level)
     logger = c.logger
-    logger.info("collator: ", str(args))
 
     meta = c.validate()
 

src/xchemalign/aligner.py

@@ -194,15 +194,16 @@ def get_datasets_from_crystals(
 
 
 class Aligner:
-    def __init__(self, dir, logger=None):
+    def __init__(self, dir, log_file=None, log_level=0):
         self.num_alignments = 0
         self.errors = []
         self.warnings = []
-        if logger:
-            self.logger = logger
-        else:
-            self.logger = utils.Logger()
+
         self._find_version_dir(dir)  # sets self.working_dir and self.version_dir
+        if not log_file:
+            log_file = self.working_dir / 'upload-current' / 'aligner.log'
+        self.logger = utils.Logger(logfile=log_file, level=log_level)
+        self.logger.info("Using", self.version_dir, "as current version dir")
         self.base_dir = self.version_dir.parent  # e.g. path/to
         self.aligned_dir = (
             self.version_dir / Constants.META_ALIGNED_FILES
@@ -246,8 +247,6 @@ def _find_version_dir(self, dir):
             else:
                 break
 
-        self.logger.info("Using", version_dir, "as current version dir")
-
     def _log_error(self, msg):
         self.logger.error(msg)
         self.errors.append(msg)
@@ -934,13 +933,11 @@ def main():
     else:
         log = 'aligner.log'
 
-    logger = utils.Logger(logfile=log, level=args.log_level)
-    logger.info("aligner: ", args)
-    logger.info("Using {} for log file".format(str(log)))
-    utils.LOG = logger
-
     try:
-        a = Aligner(args.dir, logger=logger)
+        a = Aligner(args.dir, log_file=log, log_level=args.log_level)
+        logger = a.logger
+        utils.LOG = logger
+
         num_errors, num_warnings = a.validate()
 
         if not args.validate:

src/xchemalign/collator.py

@@ -1385,7 +1385,6 @@ def main():
     parser = argparse.ArgumentParser(description="collator")
 
     parser.add_argument("-d", "--dir", help="Working directory")
-    parser.add_argument("-l", "--log-file", help="File to write logs to")
     parser.add_argument("--log-level", type=int, default=0, help="Logging level")
     parser.add_argument("-v", "--validate", action="store_true", help="Only perform validation")
     parser.add_argument("--no-git-info", action="store_false", help="Don't add GIT info to metadata")
@@ -1412,7 +1411,7 @@ def main():
 
     logger = None
     try:
-        c = Collator(working_dir, log_file=args.log_file, log_level=args.log_level, include_git_info=args.no_git_info)
+        c = Collator(working_dir, log_level=args.log_level, include_git_info=args.no_git_info)
 
         logger = c.logger
         logger.info("collator: ", str(args))