Merge branch 'issue_1624_tests' into uploader

Author: kaliif

.gitignore

@@ -9,8 +9,13 @@ venv/
 xchemalign/xchem_align.egg-info/
 xchemalign/_version.py
 test-data/outputs/upload_*
+test-data/outputs/upload-v2/upload_*
+test-data/outputs/upload-v2/*.log
+test-data/outputs/upload-v2/config.yaml
+
+example-simple
+build
+
 *~
 \#*#
 \#*
-example-simple
-build

scripts/align.py

@@ -11,28 +11,17 @@
 def main():
     parser = argparse.ArgumentParser(description="aligner")
 
-    parser.add_argument("-d", "--version-dir", required=True, help="Path to version dir")
-    parser.add_argument(
-        "-m", "--metadata_file", default=Constants.METADATA_XTAL_FILENAME.format(""), help="Metadata YAML file"
-    )
-    parser.add_argument("-a", "--assemblies", help="Assemblies YAML file")
+    parser.add_argument("-d", "--dir", help="Working directory")
 
-    parser.add_argument("-l", "--log-file", help="File to write logs to")
     parser.add_argument("--log-level", type=int, default=0, help="Logging level")
     parser.add_argument("--validate", action="store_true", help="Only perform validation")
 
     args = parser.parse_args()
 
-    if args.log_file:
-        log = args.log_file
-    else:
-        log = str(Path(args.version_dir).parent / 'aligner.log')
-        print("Using {} for log file".format(str(log)))
+    print("aligner: ", args)
 
-    logger = utils.Logger(logfile=log, level=args.log_level)
-    logger.info("aligner: ", args)
-
-    a = Aligner(args.version_dir, args.metadata_file, args.assemblies, logger=logger)
+    a = Aligner(args.dir, log_level=args.log_level)
+    logger = a.logger
     num_errors, num_warnings = a.validate()
 
     if not args.validate:

scripts/collate.py

@@ -8,16 +8,16 @@
 def main():
     parser = argparse.ArgumentParser(description="collator")
 
-    parser.add_argument("-c", "--config-file", default="config.yaml", help="Configuration file")
-    parser.add_argument("-l", "--log-file", help="File to write logs to")
+    parser.add_argument("-d", "--dir", help="Working directory")
     parser.add_argument("--log-level", type=int, default=0, help="Logging level")
     parser.add_argument("-v", "--validate", action="store_true", help="Only perform validation")
 
     args = parser.parse_args()
 
-    c = Collator(args.config_file, log_file=args.log_file, log_level=args.log_level)
+    print("collator: ", str(args))
+
+    c = Collator(args.dir, log_level=args.log_level)
     logger = c.logger
-    logger.info("collator: ", str(args))
 
     meta = c.validate()
 

src/xchemalign/aligner.py

@@ -194,15 +194,16 @@ def get_datasets_from_crystals(
 
 
 class Aligner:
-    def __init__(self, dir, logger=None):
+    def __init__(self, dir, log_file=None, log_level=0):
         self.num_alignments = 0
         self.errors = []
         self.warnings = []
-        if logger:
-            self.logger = logger
-        else:
-            self.logger = utils.Logger()
+
         self._find_version_dir(dir)  # sets self.working_dir and self.version_dir
+        if not log_file:
+            log_file = self.working_dir / 'upload-current' / 'aligner.log'
+        self.logger = utils.Logger(logfile=log_file, level=log_level)
+        self.logger.info("Using", self.version_dir, "as current version dir")
         self.base_dir = self.version_dir.parent  # e.g. path/to
         self.aligned_dir = (
             self.version_dir / Constants.META_ALIGNED_FILES
@@ -246,8 +247,6 @@ def _find_version_dir(self, dir):
             else:
                 break
 
-        self.logger.info("Using", version_dir, "as current version dir")
-
     def _log_error(self, msg):
         self.logger.error(msg)
         self.errors.append(msg)
@@ -934,13 +933,11 @@ def main():
     else:
         log = 'aligner.log'
 
-    logger = utils.Logger(logfile=log, level=args.log_level)
-    logger.info("aligner: ", args)
-    logger.info("Using {} for log file".format(str(log)))
-    utils.LOG = logger
-
     try:
-        a = Aligner(args.dir, logger=logger)
+        a = Aligner(args.dir, log_file=log, log_level=args.log_level)
+        logger = a.logger
+        utils.LOG = logger
+
         num_errors, num_warnings = a.validate()
 
         if not args.validate:

src/xchemalign/collator.py

@@ -1385,7 +1385,6 @@ def main():
     parser = argparse.ArgumentParser(description="collator")
 
     parser.add_argument("-d", "--dir", help="Working directory")
-    parser.add_argument("-l", "--log-file", help="File to write logs to")
     parser.add_argument("--log-level", type=int, default=0, help="Logging level")
     parser.add_argument("-v", "--validate", action="store_true", help="Only perform validation")
     parser.add_argument("--no-git-info", action="store_false", help="Don't add GIT info to metadata")
@@ -1412,7 +1411,7 @@ def main():
 
     logger = None
     try:
-        c = Collator(working_dir, log_file=args.log_file, log_level=args.log_level, include_git_info=args.no_git_info)
+        c = Collator(working_dir, log_level=args.log_level, include_git_info=args.no_git_info)
 
         logger = c.logger
         logger.info("collator: ", str(args))

test-data/outputs/upload-current

@@ -0,0 +1 @@
+upload-v2

test-data/outputs/assemblies.yaml test-data/outputs/upload-v2/assemblies.yaml

@@ -15,11 +15,11 @@ class Constants:
         CONFIG_1_FILE = "test-data/config_1.yaml"
         CONFIG_2_FILE = "test-data/config_2.yaml"
         CONFIG_3_FILE = "test-data/config_3.yaml"
-        ASSEMBLIES_FILE = "test-data/outputs/assemblies.yaml"
         TEST_DIR = "test-data/outputs"
-        UPLOAD_1_DIR = "test-data/outputs/upload_1"
-        UPLOAD_2_DIR = "test-data/outputs/upload_2"
-        UPLOAD_3_DIR = "test-data/outputs/upload_3"
+        CONFIG_FILE = TEST_DIR + '/upload-current/config.yaml'
+        UPLOAD_1_DIR = TEST_DIR + '/upload-current/upload_1'
+        UPLOAD_2_DIR = TEST_DIR + '/upload-current/upload_2'
+        UPLOAD_3_DIR = TEST_DIR + '/upload-current/upload_3'
         METADATA_FILE = "meta_collator.yaml"
 
     return Constants()
@@ -50,19 +50,19 @@ def test_dir(constants):
 
 @pytest.fixture(scope="session")
 def upload_1_dir(constants, test_dir):
-    path = Path(constants.UPLOAD_1_DIR)
-    if path.exists():
-        shutil.rmtree(path)
+    for path in [Path(constants.UPLOAD_3_DIR), Path(constants.UPLOAD_2_DIR), Path(constants.UPLOAD_1_DIR)]:
+        if path.exists():
+            shutil.rmtree(path)
     os.mkdir(path)
 
     return path
 
 
 @pytest.fixture(scope="session")
 def upload_2_dir(constants, test_dir):
-    path = Path(constants.UPLOAD_2_DIR)
-    if path.exists():
-        shutil.rmtree(path)
+    for path in [Path(constants.UPLOAD_3_DIR), Path(constants.UPLOAD_2_DIR)]:
+        if path.exists():
+            shutil.rmtree(path)
     os.mkdir(path)
 
     return path
@@ -78,59 +78,28 @@ def upload_3_dir(constants, test_dir):
     return path
 
 
-@pytest.fixture(scope="session")
-def upload_1_data_dir(constants, test_dir):
-    path = Path(constants.INPUT_1_DIR)
-    return path
-
-
-@pytest.fixture(scope="session")
-def upload_2_data_dir(constants, test_dir):
-    path = Path(constants.INPUT_2_DIR)
-    return path
-
-
-@pytest.fixture(scope="session")
-def upload_3_data_dir(constants, test_dir):
-    path = Path(constants.INPUT_3_DIR)
-    return path
-
-
 @pytest.fixture(scope="session")
 def config_1_file(
     constants,
 ):
-    path = Path(constants.CONFIG_1_FILE)
+    path = Path(constants.CONFIG_FILE)
+    shutil.copy(constants.CONFIG_1_FILE, constants.CONFIG_FILE)
     return path
 
 
 @pytest.fixture(scope="session")
 def config_2_file(
     constants,
 ):
-    path = Path(constants.CONFIG_2_FILE)
+    path = Path(constants.CONFIG_FILE)
+    shutil.copy(constants.CONFIG_2_FILE, constants.CONFIG_FILE)
     return path
 
 
 @pytest.fixture(scope="session")
 def config_3_file(
     constants,
 ):
-    path = Path(constants.CONFIG_3_FILE)
-    return path
-
-
-@pytest.fixture(scope="session")
-def assemblies_file(
-    constants,
-):
-    path = Path(constants.ASSEMBLIES_FILE)
-    return path
-
-
-@pytest.fixture(scope="session")
-def xtalforms_file(
-    constants,
-):
-    path = Path(constants.XTAFLORMS_FILE)
+    path = Path(constants.CONFIG_FILE)
+    shutil.copy(constants.CONFIG_3_FILE, constants.CONFIG_FILE)
     return path

tests/conftest.py

@@ -8,16 +8,8 @@
 from xchemalign.utils import Constants
 
 
-def test_collator_upload_1(
-    constants,
-    test_dir,
-    upload_1_dir,
-    upload_1_data_dir,
-    config_1_file,
-):
-    c = Collator(
-        str(config_1_file),
-    )
+def test_collator_upload_1(constants, test_dir, config_1_file, upload_1_dir):
+    c = Collator(constants.TEST_DIR)
     logger = c.logger
 
     meta = c.validate()
@@ -31,8 +23,8 @@ def test_collator_upload_1(
 
 
 @pytest.mark.order(after="test_collator_upload_1")
-def test_aligner_upload_1(constants, assemblies_file, upload_1_dir):
-    a = Aligner(upload_1_dir, constants.METADATA_FILE, assemblies_file)
+def test_aligner_upload_1(constants):
+    a = Aligner(constants.TEST_DIR)
     num_errors, num_warnings = a.validate()
 
     if num_errors:
@@ -44,35 +36,34 @@ def test_aligner_upload_1(constants, assemblies_file, upload_1_dir):
 
 
 @pytest.mark.order(after="test_aligner_upload_1")
-def test_collator_upload_2(
-    constants,
-    test_dir,
-    upload_2_dir,
-    upload_2_data_dir,
-    config_2_file,
-):
-    c = Collator(
-        str(config_2_file),
-    )
+def test_collator_upload_2(constants, config_2_file, upload_2_dir):
+    c = Collator(constants.TEST_DIR)
     logger = c.logger
 
     meta = c.validate()
 
+    print(meta)
+
     if meta is None or len(logger.errors) > 0:
         print("There are errors, cannot continue")
         print(logger.errors)
         exit(1)
     else:
         c.run(meta)
     assert (
-        len(meta[Constants.META_XTALS]["Mpro-i0130"][Constants.META_XTAL_FILES].get(Constants.META_BINDING_EVENT, {}))
+        # len(meta[Constants.META_XTALS]["Mpro-i0130"][Constants.META_XTAL_FILES].get(Constants.META_BINDING_EVENT, {}))
+        len(
+            meta[Constants.META_XTALS]["Mpro-x0107_fake_P1"][Constants.META_XTAL_FILES].get(
+                Constants.META_BINDING_EVENT, {}
+            )
+        )
         != 0
     )
 
 
 @pytest.mark.order(after="test_collator_upload_2")
-def test_aligner_upload_2(constants, assemblies_file, upload_2_dir):
-    a = Aligner(upload_2_dir, constants.METADATA_FILE, assemblies_file)
+def test_aligner_upload_2(constants):
+    a = Aligner(constants.TEST_DIR)
     num_errors, num_warnings = a.validate()
 
     if num_errors:
@@ -81,7 +72,7 @@ def test_aligner_upload_2(constants, assemblies_file, upload_2_dir):
         exit(1)
     else:
         a.run()
-    assert "Mpro-i0130" in [x.name for x in (Path(upload_2_dir) / "aligned_files").glob("*")]
+    assert "Mpro-i0130" in [x.name for x in (Path(constants.UPLOAD_2_DIR) / "aligned_files").glob("*")]
 
 
 # @pytest.mark.order(after="test_aligner_upload_2")