From 0831ffceb704723981a8e4ce943f145d5f4ee275 Mon Sep 17 00:00:00 2001 From: Per Jensen Date: Fri, 3 Jan 2025 13:17:14 +0100 Subject: [PATCH] manager: --add-specific-archive added --- v2/src/dar_backup/manager.py | 163 +++++++++++++++++++++++------------ v2/src/dar_backup/util.py | 2 +- v2/tests/conftest.py | 2 +- v2/tests/test_manager.py | 67 +++++++++++++- 4 files changed, 175 insertions(+), 59 deletions(-) diff --git a/v2/src/dar_backup/manager.py b/v2/src/dar_backup/manager.py index 6ed75a2..4271012 100644 --- a/v2/src/dar_backup/manager.py +++ b/v2/src/dar_backup/manager.py @@ -23,7 +23,6 @@ import os import argparse -import subprocess import sys @@ -32,77 +31,127 @@ from dar_backup.util import run_command from dar_backup.util import setup_logging from time import time + # Constants SCRIPTNAME = os.path.basename(__file__) SCRIPTPATH = os.path.realpath(__file__) SCRIPTDIRPATH = os.path.dirname(SCRIPTPATH) DB_SUFFIX = ".db" -# Helper functions -def show_help(): +logger = None + +def show_more_help(): help_text = f""" NAME - {SCRIPTNAME} - creates/maintains dar catalogs for dar archives for backup definitions + {SCRIPTNAME} - creates/maintains `dar` databases with catalogs for backup definitions """ print(help_text) -def log_error(message): - print(f"ERROR: {message}", file=sys.stderr) - - def create_db(backup_def: str, config_settings: ConfigSettings): - catalog = f"{backup_def}{DB_SUFFIX}" + database = f"{backup_def}{DB_SUFFIX}" - catalog_path = os.path.join(config_settings.backup_dir, catalog) + database_path = os.path.join(config_settings.backup_dir, database) - print(f"backups dir: {config_settings.backup_dir}") - print(f"catalog: {catalog_path}") + logger.debug(f"BACKUP_DIR: {config_settings.backup_dir}") - if os.path.exists(catalog_path): - logger.warning(f'"{catalog_path}" already exists, skipping creation') + if os.path.exists(database_path): + logger.warning(f'"{database_path}" already exists, skipping creation') else: - logger.info(f'Create catalog database: "{catalog_path}"') - command = ['dar_manager', '--create' , f"{catalog_path}"] + logger.info(f'Create catalog database: "{database_path}"') + command = ['dar_manager', '--create' , database_path] process = run_command(command) - stdout, stderr = process.communicate() - print (f"out: {stdout}") - print (f"err: {stderr}") - print (f"return code from 'db created': {process.returncode}") + logger.debug(f"return code from 'db created': {process.returncode}") if process.returncode == 0: - logger.info(f'Catalog created: "{catalog_path}"') + logger.info(f'Database created: "{database_path}"') else: - logger.error(f'Something went wrong creating the catalog: "{catalog_path}"') + logger.error(f'Something went wrong creating the database: "{database_path}"') + stdout, stderr = process.communicate() logger.error(f"stderr: {stderr}") + logger.error(f"stdout: {stdout}") + + + +def list_db(backup_def: str, config_settings: ConfigSettings): + database = f"{backup_def}{DB_SUFFIX}" + database_path = os.path.join(config_settings.backup_dir, database) + if not os.path.exists(database_path): + logger.error(f'Database not found: "{database_path}"') + return 1 + command = ['dar_manager', '--base', database_path, '--list'] + process = run_command(command) + stdout, stderr = process.communicate() + if process.returncode != 0: + logger.error(f'Error listing catalogs for: "{database_path}"') + logger.error(f"stderr: {stderr}") + logger.error(f"stdout: {stdout}") + else: + print(stdout) + sys.exit(process.returncode) + + +def add_specific_archive(archive: str, config_settings: ConfigSettings): + # sanity check - does dar backup exist? + archive = os.path.basename(archive) # remove path if it was given + archive_path = os.path.join(config_settings.backup_dir, f'{archive}.1.dar') + if not os.path.exists(archive_path): + logger.error(f'dar backup: "{archive_path}" not found, exiting') + sys.exit(1) + + # sanity check - does backup definition exist? + backup_definition = archive.split('_')[0] + backup_def_path = os.path.join(config_settings.backup_d_dir, backup_definition) + if not os.path.exists(backup_def_path): + logger.error(f'backup definition "{backup_definition}" not found (--add-specific-archive option probably not correct), exiting') + sys.exit(1) + + database = f"{backup_definition}{DB_SUFFIX}" + database_path = os.path.realpath(os.path.join(config_settings.backup_dir, database)) + logger.info(f'Add "{archive_path}" to catalog "{database}"') + + command = ['dar_manager', '--base', database_path, "--add", archive_path, "-ai", "-Q"] + process = run_command(command) + stdout, stderr = process.communicate() + + if process.returncode == 0: + logger.info(f'"{archive_path}" added to it\'s catalog') + elif process.returncode == 5: + logger.warning(f'Something did not go completely right adding "{archive_path}" to it\'s catalog, dar_manager error: "{process.returncode}"') + else: + logger.error(f'something went wrong adding "{archive_path}" to it\'s catalog, dar_manager error: "{process.returncode}"') + logger.error(f"stderr: {stderr}") + logger.error(f"stdout: {stdout}") + + sys.exit(process.returncode) + def main(): + global logger + MIN_PYTHON_VERSION = (3, 9) if sys.version_info < MIN_PYTHON_VERSION: sys.stderr.write(f"Error: This script requires Python {'.'.join(map(str, MIN_PYTHON_VERSION))} or higher.\n") sys.exit(1) - - global logger - - parser = argparse.ArgumentParser(description="Creates/maintains dar catalogs for dar archives for backup definitions") + parser = argparse.ArgumentParser(description="Creates/maintains `dar` database catalogs") parser.add_argument('-c', '--config-file', type=str, help="Path to 'dar-backup.conf'", default='~/.config/dar-backup/dar-backup.conf') - parser.add_argument('--create-db', action='store_true', help='Create missing catalogs') - parser.add_argument('--alternate-archive-dir', type=str, help='Work on this dir instead of MOUNT_POINT') - parser.add_argument('--add-dir', type=str, help='Add all archives in a dir') - parser.add_argument('-d', '--backup-def', type=str, help='Restrict to add only archives for this backup definition') - parser.add_argument('--add-specific-archive', type=str, help='Add this archive to catalog') - parser.add_argument('--remove-specific-archive', type=str, help='Remove this archive from catalog') - parser.add_argument('--list-db', action='store_true', help='List db for catalogs') - parser.add_argument('--verbose', action='store_true', help='Output a single notice on adding an archive to its catalog') - parser.add_argument('--log-level', type=str, help="`debug` or `trace`", default="info") - parser.add_argument('--added-help', action='store_true', help='Show help message and exit') - parser.add_argument('--version', action='store_true', help='Show version and exit') + parser.add_argument('--create-db', action='store_true', help='Create missing databases for all backup definitions') + parser.add_argument('--alternate-archive-dir', type=str, help='Use this directory instead of BACKUP_DIR in config file') + parser.add_argument('--add-dir', type=str, help='Add all archive catalogs in this directory to databases') + parser.add_argument('-d', '--backup-def', type=str, help='Restrict to work only on this backup definition') + parser.add_argument('--add-specific-archive', type=str, help='Add this archive to catalog database') + parser.add_argument('--remove-specific-archive', type=str, help='Remove this archive from catalog database') + parser.add_argument('--list-db', action='store_true', help='List catalogs in databases') + parser.add_argument('--verbose', action='store_true', help='Be more verbose') + parser.add_argument('--log-level', type=str, help="`debug` or `trace`, default is `info`", default="info") + parser.add_argument('--more-help', action='store_true', help='Show extended help message') + parser.add_argument('--version', action='store_true', help='Show version & license') args = parser.parse_args() - if args.added_help: - show_help() + if args.more_help: + show_more_help() sys.exit(0) if args.version: @@ -113,11 +162,11 @@ def main(): See section 15 and section 16 in the supplied "LICENSE" file.''') sys.exit(0) - + # setup logging args.config_file = os.path.expanduser(args.config_file) config_settings = ConfigSettings(args.config_file) - if not config_settings.logfile_location or not os.path.dirname(config_settings.logfile_location): - print(f"Log file '{config_settings.logfile_location}' does not exist, exiting") + if not os.path.dirname(config_settings.logfile_location): + print(f"Directory for log file '{config_settings.logfile_location}' does not exist, exiting") sys.exit(1) logger = setup_logging(config_settings.logfile_location, args.log_level) @@ -131,38 +180,36 @@ def main(): # Sanity checks before starting if args.add_dir and not args.add_dir.strip(): - log_error("archive dir not given, exiting") + logger.error("archive dir not given, exiting") sys.exit(1) if args.add_specific_archive and not args.add_specific_archive.strip(): - log_error("specific archive to add not given, exiting") + logger.error("specific archive to add not given, exiting") sys.exit(1) if args.remove_specific_archive and not args.remove_specific_archive.strip(): - log_error("specific archive to remove not given, exiting") + logger.error("specific archive to remove not given, exiting") sys.exit(1) if args.add_specific_archive and args.remove_specific_archive: - log_error("you can't add and remove archives in the same operation, exiting") + logger.error("you can't add and remove archives in the same operation, exiting") sys.exit(1) if args.backup_def: backup_def_path = os.path.join(config_settings.backup_d_dir, args.backup_def) if not os.path.exists(backup_def_path): - log_error(f"Backup definition {args.backup_def} does not exist, exiting") + logger.error(f"Backup definition {args.backup_def} does not exist, exiting") sys.exit(1) # Modify config settings based on the arguments if args.alternate_archive_dir: if not os.path.exists(args.alternate_archive_dir): - log_error(f"Alternate archive dir '{args.alternate_archive_dir}' does not exist, exiting") + logger.error(f"Alternate archive dir '{args.alternate_archive_dir}' does not exist, exiting") sys.exit(1) config_settings.backup_dir = args.alternate_archive_dir - - # Implement the logic for the operations based on the arguments if args.create_db: if args.backup_def: create_db(args.backup_def, config_settings) @@ -174,8 +221,7 @@ def main(): sys.exit(0) if args.add_specific_archive: - # Implement add specific archive logic - pass + add_specific_archive(args.add_specific_archive, config_settings) if args.add_dir: # Implement add directory logic @@ -185,9 +231,16 @@ def main(): # Implement remove specific archive logic pass - if args.list_catalog: - # Implement list catalog logic - pass + + if args.list_db: + if args.backup_def: + list_db(args.backup_def, config_settings) + else: + for root, dirs, files in os.walk(config_settings.backup_d_dir): + for file in files: + current_backupdef = os.path.basename(file) + list_db(current_backupdef, config_settings) + sys.exit(0) if __name__ == "__main__": main() diff --git a/v2/src/dar_backup/util.py b/v2/src/dar_backup/util.py index 0951886..c0ae50c 100644 --- a/v2/src/dar_backup/util.py +++ b/v2/src/dar_backup/util.py @@ -94,7 +94,7 @@ def run_command(command: list[str]) -> subprocess.CompletedProcess: try: logger.trace(f"Running command: {command}") process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - process.wait() + process.wait() # Popen() and wait() used, for potential future use of stdout and stderr while a subprocess is running. except Exception as e: logger.error(f"Error running command: {command}", exc_info=True) raise diff --git a/v2/tests/conftest.py b/v2/tests/conftest.py index 66105f2..c322201 100644 --- a/v2/tests/conftest.py +++ b/v2/tests/conftest.py @@ -81,7 +81,7 @@ def setup_environment(request): # in the code below. # Tear down the environment after the test case try: - #teardown_environment(env) + teardown_environment(env) pass except Exception as e: env.logger.exception("Failed to tear down environment") diff --git a/v2/tests/test_manager.py b/v2/tests/test_manager.py index f0998a3..cd1d71d 100644 --- a/v2/tests/test_manager.py +++ b/v2/tests/test_manager.py @@ -5,13 +5,16 @@ import re import dar_backup.config_settings import envdata +import test_bitrot +from datetime import date from dar_backup.util import run_command + def generate_3_backup_defs(env, config_settings): """ - Generate 3 backup definitions + Generate 3 dummy backup definitions """ for i in range(3): backup_def = f"test{i}" @@ -69,10 +72,70 @@ def test_manager_version(setup_environment, env): def test_manager_help(setup_environment, env): - command = ['manager', '--added-help'] + command = ['manager', '--more-help'] + process = run_command(command) + if process.returncode != 0: + stdout, stderr = process.communicate() + print(f"stdout: {stdout}") + print(f"stderr: {stderr}") + raise Exception(f"Command failed: {command}") + + +def test_manager_add_specific_archive(setup_environment, env): + + today_date = date.today().strftime("%Y-%m-%d") + + file_sizes = { + '1byte': 1, + '10bytes': 10, + '100bytes': 100, + '1000bytes': 1000, + '10kB': 10 * 1024, + '100kB': 100 * 1024, + '1MB': 1024 * 1024, + '10MB': 10 * 1024 * 1024 + } + test_bitrot.generate_datafiles(env, file_sizes) + command = ['dar-backup', '--full-backup' ,'-d', "example", '--config-file', env.config_file] + process = run_command(command) + stdout,stderr = process.communicate() + if process.returncode != 0: + print(f"dar stdout: {stdout}") + print(f"dar stderr: {stderr}") + raise RuntimeError(f"dar-backup failed to create a full backup") + + + # generate database for catalogs + command = ['manager', '--create-db' ,'--config-file', env.config_file] process = run_command(command) if process.returncode != 0: stdout, stderr = process.communicate() print(f"stdout: {stdout}") print(f"stderr: {stderr}") raise Exception(f"Command failed: {command}") + + # add archive to catalog + command = ['manager', '--add-specific-archive' ,f'example_FULL_{today_date}', '--config-file', env.config_file, '--log-level', "trace"] + process = run_command(command) + if process.returncode != 0: + stdout, stderr = process.communicate() + print(f"stdout: {stdout}") + print(f"stderr: {stderr}") + raise Exception(f"Command failed: {command}") + + # list catalogs + command = ['manager', '--list-db' ,'--config-file', env.config_file] + process = run_command(command) + stdout, stderr = process.communicate() + if process.returncode != 0: + print(f"stdout: {stdout}") + print(f"stderr: {stderr}") + raise Exception(f"Command failed: {command}") + + print(f"stdout: {stdout}") + + if not re.search(f"example_FULL_{today_date}", stdout): + raise Exception(f"Catalog not found for backup definition f'example_FULL_{today_date}'") + + print(f"Catalog for example_FULL_{today_date}.1.dar found in example.db") +