manager: --add-specific-archive added

v2/src/dar_backup/manager.py

@@ -23,7 +23,6 @@
 
 import os
 import argparse
-import subprocess
 import sys
 
 
@@ -32,77 +31,127 @@
 from dar_backup.util import run_command
 from dar_backup.util import setup_logging
 from time import time
+
 # Constants
 SCRIPTNAME = os.path.basename(__file__)
 SCRIPTPATH = os.path.realpath(__file__)
 SCRIPTDIRPATH = os.path.dirname(SCRIPTPATH)
 DB_SUFFIX = ".db"
 
-# Helper functions
-def show_help():
+logger = None
+
+def show_more_help():
     help_text = f"""
 NAME
-    {SCRIPTNAME} - creates/maintains dar catalogs for dar archives for backup definitions
+    {SCRIPTNAME} - creates/maintains `dar` databases with catalogs for backup definitions
 """
     print(help_text)
 
 
-def log_error(message):
-    print(f"ERROR: {message}", file=sys.stderr)
-
-
 def create_db(backup_def: str, config_settings: ConfigSettings):
-    catalog = f"{backup_def}{DB_SUFFIX}"
+    database = f"{backup_def}{DB_SUFFIX}"
 
-    catalog_path = os.path.join(config_settings.backup_dir, catalog)
+    database_path = os.path.join(config_settings.backup_dir, database)
 
-    print(f"backups dir: {config_settings.backup_dir}")
-    print(f"catalog: {catalog_path}")
+    logger.debug(f"BACKUP_DIR: {config_settings.backup_dir}")
 
-    if os.path.exists(catalog_path):
-        logger.warning(f'"{catalog_path}" already exists, skipping creation')
+    if os.path.exists(database_path):
+        logger.warning(f'"{database_path}" already exists, skipping creation')
     else:
-        logger.info(f'Create catalog database: "{catalog_path}"')
-        command = ['dar_manager', '--create' , f"{catalog_path}"]
+        logger.info(f'Create catalog database: "{database_path}"')
+        command = ['dar_manager', '--create' , database_path]
         process = run_command(command)
-        stdout, stderr = process.communicate()
-        print (f"out: {stdout}")
-        print (f"err: {stderr}")
-        print (f"return code from 'db created': {process.returncode}")
+        logger.debug(f"return code from 'db created': {process.returncode}")
         if process.returncode == 0:
-            logger.info(f'Catalog created: "{catalog_path}"')
+            logger.info(f'Database created: "{database_path}"')
         else:
-            logger.error(f'Something went wrong creating the catalog: "{catalog_path}"')
+            logger.error(f'Something went wrong creating the database: "{database_path}"')
+            stdout, stderr = process.communicate()
             logger.error(f"stderr: {stderr}")
+            logger.error(f"stdout: {stdout}")
+
+
+
+def list_db(backup_def: str, config_settings: ConfigSettings):
+    database = f"{backup_def}{DB_SUFFIX}"
+    database_path = os.path.join(config_settings.backup_dir, database)
+    if not os.path.exists(database_path):
+        logger.error(f'Database not found: "{database_path}"')
+        return 1
+    command = ['dar_manager', '--base', database_path, '--list']
+    process = run_command(command)
+    stdout, stderr = process.communicate()
+    if process.returncode != 0:
+        logger.error(f'Error listing catalogs for: "{database_path}"')
+        logger.error(f"stderr: {stderr}")  
+        logger.error(f"stdout: {stdout}")
+    else:
+        print(stdout)
+    sys.exit(process.returncode)
+
+
+def add_specific_archive(archive: str, config_settings: ConfigSettings):    
+    # sanity check - does dar backup exist?
+    archive = os.path.basename(archive)  # remove path if it was given
+    archive_path = os.path.join(config_settings.backup_dir, f'{archive}.1.dar')
+    if not os.path.exists(archive_path):
+        logger.error(f'dar backup: "{archive_path}" not found, exiting')
+        sys.exit(1)
+
+    # sanity check - does backup definition exist?
+    backup_definition = archive.split('_')[0]
+    backup_def_path = os.path.join(config_settings.backup_d_dir, backup_definition)
+    if not os.path.exists(backup_def_path):
+        logger.error(f'backup definition "{backup_definition}" not found (--add-specific-archive option probably not correct), exiting')
+        sys.exit(1)
+
+    database = f"{backup_definition}{DB_SUFFIX}"
+    database_path = os.path.realpath(os.path.join(config_settings.backup_dir, database))
+    logger.info(f'Add "{archive_path}" to catalog "{database}"')
+
+    command = ['dar_manager', '--base', database_path, "--add", archive_path, "-ai", "-Q"]
+    process = run_command(command)
+    stdout, stderr = process.communicate()
+
+    if process.returncode == 0:
+        logger.info(f'"{archive_path}" added to it\'s catalog')
+    elif process.returncode == 5:
+        logger.warning(f'Something did not go completely right adding "{archive_path}" to it\'s catalog, dar_manager error: "{process.returncode}"')
+    else: 
+        logger.error(f'something went wrong adding "{archive_path}" to it\'s catalog, dar_manager error: "{process.returncode}"')
+        logger.error(f"stderr: {stderr}")
+        logger.error(f"stdout: {stdout}")
+
+    sys.exit(process.returncode)
+
 
 
 def main():
+    global logger 
+
     MIN_PYTHON_VERSION = (3, 9)
     if sys.version_info < MIN_PYTHON_VERSION:
         sys.stderr.write(f"Error: This script requires Python {'.'.join(map(str, MIN_PYTHON_VERSION))} or higher.\n")
         sys.exit(1)
 
-
-    global logger 
-
-    parser = argparse.ArgumentParser(description="Creates/maintains dar catalogs for dar archives for backup definitions")
+    parser = argparse.ArgumentParser(description="Creates/maintains `dar` database catalogs")
     parser.add_argument('-c', '--config-file', type=str, help="Path to 'dar-backup.conf'", default='~/.config/dar-backup/dar-backup.conf')
-    parser.add_argument('--create-db', action='store_true', help='Create missing catalogs')
-    parser.add_argument('--alternate-archive-dir', type=str, help='Work on this dir instead of MOUNT_POINT')
-    parser.add_argument('--add-dir', type=str, help='Add all archives in a dir')
-    parser.add_argument('-d', '--backup-def', type=str, help='Restrict to add only archives for this backup definition')
-    parser.add_argument('--add-specific-archive', type=str, help='Add this archive to catalog')
-    parser.add_argument('--remove-specific-archive', type=str, help='Remove this archive from catalog')
-    parser.add_argument('--list-db', action='store_true', help='List db for catalogs')
-    parser.add_argument('--verbose', action='store_true', help='Output a single notice on adding an archive to its catalog')
-    parser.add_argument('--log-level', type=str, help="`debug` or `trace`", default="info")
-    parser.add_argument('--added-help', action='store_true', help='Show help message and exit')
-    parser.add_argument('--version', action='store_true', help='Show version and exit')
+    parser.add_argument('--create-db', action='store_true', help='Create missing databases for all backup definitions')
+    parser.add_argument('--alternate-archive-dir', type=str, help='Use this directory instead of BACKUP_DIR in config file')
+    parser.add_argument('--add-dir', type=str, help='Add all archive catalogs in this directory to databases')
+    parser.add_argument('-d', '--backup-def', type=str, help='Restrict to work only on this backup definition')
+    parser.add_argument('--add-specific-archive', type=str, help='Add this archive to catalog database')
+    parser.add_argument('--remove-specific-archive', type=str, help='Remove this archive from catalog database')
+    parser.add_argument('--list-db', action='store_true', help='List catalogs in databases')
+    parser.add_argument('--verbose', action='store_true', help='Be more verbose')
+    parser.add_argument('--log-level', type=str, help="`debug` or `trace`, default is `info`", default="info")
+    parser.add_argument('--more-help', action='store_true', help='Show extended help message')
+    parser.add_argument('--version', action='store_true', help='Show version & license')
 
     args = parser.parse_args()
 
-    if args.added_help:
-        show_help()
+    if args.more_help:
+        show_more_help()
         sys.exit(0)
 
     if args.version:
@@ -113,11 +162,11 @@ def main():
 See section 15 and section 16 in the supplied "LICENSE" file.''')
         sys.exit(0)
 
-
+    # setup logging
     args.config_file = os.path.expanduser(args.config_file)
     config_settings = ConfigSettings(args.config_file)
-    if not config_settings.logfile_location or not os.path.dirname(config_settings.logfile_location):
-        print(f"Log file '{config_settings.logfile_location}' does not exist, exiting")
+    if not os.path.dirname(config_settings.logfile_location):
+        print(f"Directory for log file '{config_settings.logfile_location}' does not exist, exiting")
         sys.exit(1) 
     logger = setup_logging(config_settings.logfile_location, args.log_level)
 
@@ -131,38 +180,36 @@ def main():
 
     # Sanity checks before starting
     if args.add_dir and not args.add_dir.strip():
-        log_error("archive dir not given, exiting")
+        logger.error("archive dir not given, exiting")
         sys.exit(1)
 
     if args.add_specific_archive and not args.add_specific_archive.strip():
-        log_error("specific archive to add not given, exiting")
+        logger.error("specific archive to add not given, exiting")
         sys.exit(1)
 
     if args.remove_specific_archive and not args.remove_specific_archive.strip():
-        log_error("specific archive to remove not given, exiting")
+        logger.error("specific archive to remove not given, exiting")
         sys.exit(1)
 
     if args.add_specific_archive and args.remove_specific_archive:
-        log_error("you can't add and remove archives in the same operation, exiting")
+        logger.error("you can't add and remove archives in the same operation, exiting")
         sys.exit(1)
 
     if args.backup_def:
         backup_def_path = os.path.join(config_settings.backup_d_dir, args.backup_def)
         if not os.path.exists(backup_def_path):
-            log_error(f"Backup definition {args.backup_def} does not exist, exiting")
+            logger.error(f"Backup definition {args.backup_def} does not exist, exiting")
             sys.exit(1)
 
 
     # Modify config settings based on the arguments
     if args.alternate_archive_dir:
         if not os.path.exists(args.alternate_archive_dir):
-            log_error(f"Alternate archive dir '{args.alternate_archive_dir}' does not exist, exiting")
+            logger.error(f"Alternate archive dir '{args.alternate_archive_dir}' does not exist, exiting")
             sys.exit(1)
         config_settings.backup_dir = args.alternate_archive_dir
 
 
-
-    # Implement the logic for the operations based on the arguments
     if args.create_db:
         if args.backup_def:
             create_db(args.backup_def, config_settings)
@@ -174,8 +221,7 @@ def main():
         sys.exit(0)
 
     if args.add_specific_archive:
-        # Implement add specific archive logic
-        pass
+        add_specific_archive(args.add_specific_archive, config_settings)
 
     if args.add_dir:
         # Implement add directory logic
@@ -185,9 +231,16 @@ def main():
         # Implement remove specific archive logic
         pass
 
-    if args.list_catalog:
-        # Implement list catalog logic
-        pass
+
+    if args.list_db:
+        if args.backup_def:
+            list_db(args.backup_def, config_settings)
+        else:
+            for root, dirs, files in os.walk(config_settings.backup_d_dir):
+                for file in files:
+                    current_backupdef = os.path.basename(file)
+                    list_db(current_backupdef, config_settings)
+        sys.exit(0)
 
 if __name__ == "__main__":
     main()

v2/src/dar_backup/util.py

@@ -94,7 +94,7 @@ def run_command(command: list[str]) -> subprocess.CompletedProcess:
     try:
         logger.trace(f"Running command: {command}")
         process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-        process.wait()
+        process.wait() # Popen() and wait() used, for potential future use of stdout and stderr while a subprocess is running.
     except Exception as e:
         logger.error(f"Error running command: {command}", exc_info=True)
         raise

v2/tests/conftest.py

@@ -81,7 +81,7 @@ def setup_environment(request):
     # in the code below.
     # Tear down the environment after the test case
     try:
-        #teardown_environment(env)
+        teardown_environment(env)
         pass
     except Exception as e:
         env.logger.exception("Failed to tear down environment")

v2/tests/test_manager.py

@@ -5,13 +5,16 @@
 import re
 import dar_backup.config_settings
 import envdata
+import test_bitrot
 
+from datetime import date
 from dar_backup.util import run_command
 
 
+
 def generate_3_backup_defs(env, config_settings):
     """
-    Generate 3 backup definitions
+    Generate 3 dummy backup definitions
     """
     for i in range(3):
         backup_def = f"test{i}"
@@ -69,10 +72,70 @@ def test_manager_version(setup_environment, env):
 
 
 def test_manager_help(setup_environment, env):
-    command = ['manager', '--added-help']
+    command = ['manager', '--more-help']
+    process = run_command(command)
+    if process.returncode != 0:
+        stdout, stderr = process.communicate()
+        print(f"stdout: {stdout}")  
+        print(f"stderr: {stderr}")  
+        raise Exception(f"Command failed: {command}")
+
+
+def test_manager_add_specific_archive(setup_environment, env):
+
+    today_date = date.today().strftime("%Y-%m-%d")
+
+    file_sizes = {
+        '1byte': 1,
+        '10bytes': 10,
+        '100bytes': 100,
+        '1000bytes': 1000,
+        '10kB': 10 * 1024,
+        '100kB': 100 * 1024,
+        '1MB': 1024 * 1024,
+        '10MB': 10 * 1024 * 1024
+   }
+    test_bitrot.generate_datafiles(env, file_sizes)
+    command = ['dar-backup', '--full-backup' ,'-d', "example", '--config-file', env.config_file]
+    process = run_command(command)
+    stdout,stderr = process.communicate()
+    if process.returncode != 0:
+        print(f"dar stdout: {stdout}")
+        print(f"dar stderr: {stderr}")
+        raise RuntimeError(f"dar-backup failed to create a full backup")
+
+
+    # generate database for catalogs
+    command = ['manager', '--create-db' ,'--config-file', env.config_file]
     process = run_command(command)
     if process.returncode != 0:
         stdout, stderr = process.communicate()
         print(f"stdout: {stdout}")  
         print(f"stderr: {stderr}")  
         raise Exception(f"Command failed: {command}")
+
+    # add archive to catalog
+    command = ['manager', '--add-specific-archive' ,f'example_FULL_{today_date}', '--config-file', env.config_file, '--log-level', "trace"]
+    process = run_command(command)
+    if process.returncode != 0:
+        stdout, stderr = process.communicate()
+        print(f"stdout: {stdout}")  
+        print(f"stderr: {stderr}")  
+        raise Exception(f"Command failed: {command}")
+
+    # list catalogs
+    command = ['manager', '--list-db' ,'--config-file', env.config_file]
+    process = run_command(command)
+    stdout, stderr = process.communicate()
+    if process.returncode != 0:
+        print(f"stdout: {stdout}")  
+        print(f"stderr: {stderr}")  
+        raise Exception(f"Command failed: {command}")
+
+    print(f"stdout: {stdout}")
+
+    if not re.search(f"example_FULL_{today_date}", stdout):
+        raise Exception(f"Catalog not found for backup definition f'example_FULL_{today_date}'")
+
+    print(f"Catalog for example_FULL_{today_date}.1.dar found in example.db")  
+