task: delete metadata for intentionally missing vocabulary values

Author: Dominick Leppich

migration/lib/mets_context.py

@@ -6,7 +6,7 @@
 RECORD_PATTERN = re.compile('^(\\d+).*$')
 
 class Context:
-    def __init__(self, api, dry, verbose, continue_on_error, metadata_directory, mapping_file, preferred_mets_main_value_language, manual_id_fix, trust, enable_relation_vocabulary_column_logic):
+    def __init__(self, api, dry, verbose, continue_on_error, metadata_directory, mapping_file, preferred_mets_main_value_language, manual_id_fix, trust, enable_relation_vocabulary_column_logic, delete_missing_vocabulary_references):
         self.api = api
         self.dry = dry
         self.verbose = verbose
@@ -17,6 +17,8 @@ def __init__(self, api, dry, verbose, continue_on_error, metadata_directory, map
         self.manual_id_fix = manual_id_fix
         self.trust = trust
         self.enable_relation_vocabulary_column_logic = enable_relation_vocabulary_column_logic
+        self.delete_missing_vocabulary_references = delete_missing_vocabulary_references
+        self.removable_metadata_map = {}
         self.vocabulary_name_id_map = {}
         self.vocabulary_id_name_map = {}
         self.vocabulary_id_map = {}
@@ -133,6 +135,12 @@ def robust_find_record_id(self, parts):
             return vocabulary_id, record_id
         except:
             return None, None
+    
+    def is_removable_metadata(self, vocabulary_id, value):
+        if not vocabulary_id in self.removable_metadata_map:
+            return False
+
+        return value in self.removable_metadata_map[vocabulary_id]
 
     def log_processed(self, file):
         with open('mets_migration.log', 'a') as f:

migration/lib/mets_manipulator.py

@@ -187,8 +187,10 @@ def process_vocabulary_reference_by_value(self, node):
                         inverse_search_field='Reverse relationship'
 
                 try:
+                    # First, try to find the value in the correct column
                     new_record_id = self.ctx.api.find_record(self.ctx, vocabulary_id, value, search_field=search_field)
                 except:
+                    # If failed, try to find the value in the other column (assuming the value was stored incorrectly)
                     new_record_id = self.ctx.api.find_record(self.ctx, vocabulary_id, value, search_field=inverse_search_field)
                     old_value = node.text
                     record_data = self.ctx.api.lookup_record(new_record_id)
@@ -223,9 +225,27 @@ def process_vocabulary_reference_by_value(self, node):
 
             self.changed = True
         except Exception as e:
-            error = f'Unable to find record by value: {value}\n\t\t{e}'
-            logging.error(error)
-            self.ctx.log_issue(self.file_path, error)
+            # If this fails as well and the value is not found, remove the metadata if configured
+            if 'has no results' in e.__str__() and self.ctx.is_removable_metadata(vocabulary_id, node.text):
+                logging.warn(f'Removing node due to intentionally missing vocabulary value: "{node.text}"')
+                self.remove_metadata_node(node)
+            else:
+                error = f'Unable to find record by value: {value}\n\t\t{e}'
+                logging.error(error)
+                self.ctx.log_issue(self.file_path, error)
+
+    def remove_metadata_node(self, node):
+        parent = node.getparent()
+        if parent != None and parent.attrib['type'] == 'group':
+            node = parent
+        parent = node.getparent()
+
+        if parent == None:
+            dump_node(node)
+            raise Exception(f'Unable to remove node due to missing parent')
+
+        parent.remove(node)
+        self.changed = True
 
     def process_manual_id_reference(self, node):
         try:

migration/lib/mets_migrator.py

@@ -11,6 +11,7 @@ def __init__(self, ctx):
 
     def migrate(self):
         self.load_mapping_file()
+        self.load_delete_missing_vocabulary_references_file()
         self.mets_files = self.scan_for_mets_files()
         logging.info(f'{len(self.mets_files)} mets file(s) found!')
         logging.info(f'Start processing ...')
@@ -52,6 +53,31 @@ def load_mapping_file(self):
                 else:
                     raise Exception(f'Mapping file contains duplicate entry for old record {record_id_old}')
 
+    def load_delete_missing_vocabulary_references_file(self):
+        if self.ctx.delete_missing_vocabulary_references == None:
+            return
+
+        header = None
+        with open(self.ctx.delete_missing_vocabulary_references, 'r') as f:
+            for line in f:
+                line = line.strip()
+                if header == None:
+                    header = line
+                    if header != CSV_DELIMITER.join(['vocabulary_id', 'value']):
+                        raise Exception('Header mismatch in mapping file!')
+                    continue
+
+                parts = line.split(CSV_DELIMITER)
+                if len(parts) != 2:
+                    raise Exception(f'Wrong number of fields in line: {line}')
+                
+                vocabulary_id = int(parts[0])
+                value = parts[1]
+
+                if not vocabulary_id in self.ctx.removable_metadata_map:
+                    self.ctx.removable_metadata_map[vocabulary_id] = []
+                self.ctx.removable_metadata_map[vocabulary_id].append(value)
+
     def scan_for_mets_files(self):
         results = []
         for root, dirs, files in os.walk(self.ctx.metadata_directory):

migration/metadata-migrator.py

@@ -14,7 +14,7 @@ def main():
         args.vocabulary_server_port,
         args.vocabulary_server_token
     )
-    ctx = Context(api, args.dry, args.verbose, args.continue_on_error, args.metadata_directory, args.mapping_file, args.preferred_mets_main_value_language, args.manual_id_fix, args.trust, args.enable_relation_vocabulary_column_logic)
+    ctx = Context(api, args.dry, args.verbose, args.continue_on_error, args.metadata_directory, args.mapping_file, args.preferred_mets_main_value_language, args.manual_id_fix, args.trust, args.enable_relation_vocabulary_column_logic, args.delete_missing_vocabulary_references)
 
     try:
         migrator = MetsMigrator(ctx)
@@ -41,6 +41,7 @@ def parse_args():
     parser.add_argument('--preferred-mets-main-value-language', type=str, default='eng', help='Default language to use for mets value writing, if present and prior value invalid')
     parser.add_argument('--trust', required=False, type=str, default='ID', help='Set the data source to trust for the migration. Possible values are: "ID" and "Value". If "ID" is set, the record ID is parsed from the valueURI and used to find the migrated record. If "Value" is set, the XML elements value is used to find the newly migrated record by value. Defaults to "ID".')
     parser.add_argument('--enable-relation-vocabulary-column-logic', required=False, default=False, action='store_const', const=True, help='Activate relationship vocabulary correct column finding logic (reverse vs non-reverse, artist dictionary)')
+    parser.add_argument('--delete-missing-vocabulary-references', type=str, required=False, default=None, help='vocabulary and value mapping file defining intentionally removed vocabulary values that should be removed in the Mets files as well.')
     parser.add_argument('--manual-id-fix', type=str, default=None, help='Manually fix the record ID of elements whose name attribute matches this parameter. Caution, this must not be executed twice!')
     parser.add_argument('--log', required=False, default='INFO', help='logger level (possible values are: NOTSET, DEBUG, INFO, WARNING, ERROR, CRITICAL)')
     parser.add_argument('--verbose', required=False, default=False, action='store_const', const=True, help='verbose output')