From 1ccb3242cb65a507ed9d777c328456743a589dbe Mon Sep 17 00:00:00 2001 From: Andres Date: Tue, 20 Apr 2021 16:55:40 +0200 Subject: [PATCH 1/5] Add new uploader script --- .../Utilities/scripts/uploadConditions_v2.py | 470 ++++++++++++++++++ 1 file changed, 470 insertions(+) create mode 100755 CondCore/Utilities/scripts/uploadConditions_v2.py diff --git a/CondCore/Utilities/scripts/uploadConditions_v2.py b/CondCore/Utilities/scripts/uploadConditions_v2.py new file mode 100755 index 0000000000000..55c8f36e98139 --- /dev/null +++ b/CondCore/Utilities/scripts/uploadConditions_v2.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python +""" + +Joshua Dawes - CERN, CMS - The University of Manchester + +Upload script wrapper - controls the automatic update system. + +Note: the name of the file follows a different convention to the others because it should be the same as the current upload script name. + +Takes user arguments and passes them to the main upload module CondDBFW.uploads, once the correct version exists. + +1. Ask the server corresponding to the database we're uploading to which version of CondDBFW it has (query the /conddbfw_version/ url). +2. Decide which directory that we can write to - either the current local directory, or /tmp/random_string/. +3. Pull the commit returned from the server into the directory from step 2. +4. Invoke the CondDBFW.uploads module with the arguments given to this script. + +""" + +__version__ = 1 + +#import pycurl +import requests +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +try: + from StringIO import StringIO +except: + pass +import traceback +import sys +import os +import json +import subprocess +import argparse +import netrc +import shutil +import getpass +import errno +import sqlite3 + + +horizontal_rule = "="*60 + +def run_upload(**parameters): + """ + Imports CondDBFW.uploads and runs the upload with the upload metadata obtained. + """ + try: + import CondCore.Utilities.CondDBFW.uploads as uploads + except Exception as e: + traceback.print_exc() + exit("CondDBFW or one of its dependencies could not be imported.\n"\ + + "If the CondDBFW directory exists, you are likely not in a CMSSW environment.") + # we have CondDBFW, so just call the module with the parameters given in the command line + uploader = uploads.uploader(**parameters) + result = uploader.upload() + +def getInput(default, prompt = ''): + '''Like raw_input() but with a default and automatic strip(). + ''' + + answer = raw_input(prompt) + if answer: + return answer.strip() + + return default.strip() + + +def getInputWorkflow(prompt = ''): + '''Like getInput() but tailored to get target workflows (synchronization options). + ''' + + while True: + workflow = getInput(defaultWorkflow, prompt) + + if workflow in frozenset(['offline', 'hlt', 'express', 'prompt', 'pcl']): + return workflow + + print('Please specify one of the allowed workflows. See above for the explanation on each of them.') + + +def getInputChoose(optionsList, default, prompt = ''): + '''Makes the user choose from a list of options. + ''' + + while True: + index = getInput(default, prompt) + + try: + return optionsList[int(index)] + except ValueError: + print('Please specify an index of the list (i.e. integer).') + except IndexError: + print('The index you provided is not in the given list.') + + +def getInputRepeat(prompt = ''): + '''Like raw_input() but repeats if nothing is provided and automatic strip(). + ''' + + while True: + answer = raw_input(prompt) + if answer: + return answer.strip() + + print('You need to provide a value.') + +def runWizard(basename, dataFilename, metadataFilename): + while True: + print('''\nWizard for metadata for %s + +I will ask you some questions to fill the metadata file. For some of the questions there are defaults between square brackets (i.e. []), leave empty (i.e. hit Enter) to use them.''' % basename) + + # Try to get the available inputTags + try: + dataConnection = sqlite3.connect(dataFilename) + dataCursor = dataConnection.cursor() + dataCursor.execute('select name from sqlite_master where type == "table"') + tables = set(zip(*dataCursor.fetchall())[0]) + + # only conddb V2 supported... + if 'TAG' in tables: + dataCursor.execute('select NAME from TAG') + # In any other case, do not try to get the inputTags + else: + raise Exception() + + inputTags = dataCursor.fetchall() + if len(inputTags) == 0: + raise Exception() + inputTags = list(zip(*inputTags))[0] + + except Exception: + inputTags = [] + + if len(inputTags) == 0: + print('\nI could not find any input tag in your data file, but you can still specify one manually.') + + inputTag = getInputRepeat( + '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. BeamSpotObject_ByRun\ninputTag: ') + + else: + print('\nI found the following input tags in your SQLite data file:') + for (index, inputTag) in enumerate(inputTags): + print(' %s) %s' % (index, inputTag)) + + inputTag = getInputChoose(inputTags, '0', + '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. 0 (you select the first in the list)\ninputTag [0]: ') + + destinationDatabase = '' + ntry = 0 + while ( destinationDatabase != 'oracle://cms_orcon_prod/CMS_CONDITIONS' and destinationDatabase != 'oracle://cms_orcoff_prep/CMS_CONDITIONS' ): + if ntry==0: + inputMessage = \ + '\nWhich is the destination database where the tags should be exported? \nPossible choices: oracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \ndestinationDatabase: ' + elif ntry==1: + inputMessage = \ + '\nPlease choose one of the two valid destinations: \noracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \ +\ndestinationDatabase: ' + else: + raise Exception('No valid destination chosen. Bailing out...') + destinationDatabase = getInputRepeat(inputMessage) + ntry += 1 + + while True: + since = getInput('', + '\nWhich is the given since? (if not specified, the one from the SQLite data file will be taken -- note that even if specified, still this may not be the final since, depending on the synchronization options you select later: if the synchronization target is not offline, and the since you give is smaller than the next possible one (i.e. you give a run number earlier than the one which will be started/processed next in prompt/hlt/express), the DropBox will move the since ahead to go to the first safe run instead of the value you gave)\ne.g. 1234\nsince []: ') + if not since: + since = None + break + else: + try: + since = int(since) + break + except ValueError: + print('The since value has to be an integer or empty (null).') + + userText = getInput('', + '\nWrite any comments/text you may want to describe your request\ne.g. Muon alignment scenario for...\nuserText []: ') + + destinationTags = {} + while True: + destinationTag = getInput('', + '\nWhich is the next destination tag to be added (leave empty to stop)?\ne.g. BeamSpotObjects_PCL_byRun_v0_offline\ndestinationTag []: ') + if not destinationTag: + if len(destinationTags) == 0: + print('There must be at least one destination tag.') + continue + break + + if destinationTag in destinationTags: + print( + 'You already added this destination tag. Overwriting the previous one with this new one.') + + destinationTags[destinationTag] = { + } + + metadata = { + 'destinationDatabase': destinationDatabase, + 'destinationTags': destinationTags, + 'inputTag': inputTag, + 'since': since, + 'userText': userText, + } + + metadata = json.dumps(metadata, sort_keys=True, indent=4) + print('\nThis is the generated metadata:\n%s' % metadata) + + if getInput('n', + '\nIs it fine (i.e. save in %s and *upload* the conditions if this is the latest file)?\nAnswer [n]: ' % metadataFilename).lower() == 'y': + break + print('Saving generated metadata in %s...', metadataFilename) + with open(metadataFilename, 'wb') as metadataFile: + metadataFile.write(metadata) + +def parse_arguments(): + # read in command line arguments, and build metadata dictionary from them + parser = argparse.ArgumentParser(prog="cmsDbUpload client", description="CMS Conditions Upload Script in CondDBFW.") + + parser.add_argument("--sourceDB", type=str, help="DB to find Tags, IOVs + Payloads in.", required=False) + + # metadata arguments + parser.add_argument("--inputTag", type=str,\ + help="Tag to take IOVs + Payloads from in --sourceDB.", required=False) + parser.add_argument("--destinationTag", type=str,\ + help="Tag to copy IOVs + Payloads to in --destDB.", required=False) + parser.add_argument("--destinationDatabase", type=str,\ + help="Database to copy IOVs + Payloads to.", required=False) + parser.add_argument("--since", type=int,\ + help="Since to take IOVs from.", required=False) + parser.add_argument("--userText", type=str,\ + help="Description of --destTag (can be empty).") + + # non-metadata arguments + parser.add_argument("--metadataFile", "-m", type=str, help="Metadata file to take metadata from.", required=False) + + parser.add_argument("--debug", required=False, action="store_true") + parser.add_argument("--verbose", required=False, action="store_true") + parser.add_argument("--testing", required=False, action="store_true") + parser.add_argument("--fcsr-filter", type=str, help="Synchronization to take FCSR from for local filtering of IOVs.", required=False) + + parser.add_argument("--netrc", required=False) + + parser.add_argument("--hashToUse", required=False) + + parser.add_argument("--server", required=False) + + parser.add_argument("--review-options", required=False, action="store_true") + + parser.add_argument("--replay-file", required=False) + + command_line_data = parser.parse_args() + + if command_line_data.replay_file: + dictionary = json.loads("".join(open(command_line_data.replay_file, "r").readlines())) + command_line_data.tier0_response = dictionary["tier0_response"] + + # default is the production server, which can point to either database anyway + server_alias_to_url = { + "prep" : "https://cms-conddb-dev.cern.ch/cmsDbCondUpload/", + "dev" : "https://cms-conddb-dev.cern.ch/cmsDbCondUpload/", + "prod" : "https://cms-conddb.cern.ch/cmsDbCondUpload/", + None : "https://cms-conddb.cern.ch/cmsDbCondUpload/" + } + + # if prep, prod or None were given, convert to URLs in dictionary server_alias_to_url + # if not, assume a URL has been given and use this instead + if command_line_data.server in server_alias_to_url.keys(): + command_line_data.server = server_alias_to_url[command_line_data.server] + + # resolve destination databases + database_alias_to_connection = { + "prep": "oracle://cms_orcoff_prep/CMS_CONDITIONS", + "dev": "oracle://cms_orcoff_prep/CMS_CONDITIONS", + "prod": "oracle://cms_orcon_adg/CMS_CONDITIONS" + } + + if command_line_data.destinationDatabase in database_alias_to_connection.keys(): + command_line_data.destinationDatabase = database_alias_to_connection[command_line_data.destinationDatabase] + + # use netrc to get username and password + try: + netrc_file = command_line_data.netrc + netrc_authenticators = netrc.netrc(netrc_file).authenticators("ConditionUploader") + if netrc_authenticators == None: + print("Your netrc file must contain the key 'ConditionUploader'.") + manual_input = raw_input("Do you want to try to type your credentials? ") + if manual_input == "y": + # ask for username and password + username = raw_input("Username: ") + password = getpass.getpass("Password: ") + else: + exit() + else: + print("Read your credentials from ~/.netrc. If you want to use a different file, supply its name with the --netrc argument.") + username = netrc_authenticators[0] + password = netrc_authenticators[2] + except: + print("Couldn't obtain your credentials (either from netrc or manual input).") + exit() + + command_line_data.username = username + command_line_data.password = password + # this will be used as the final destinationTags value by all input methods + # apart from the metadata file + command_line_data.destinationTags = {command_line_data.destinationTag:{}} + + """ + Construct metadata_dictionary: + Currently, this is 3 cases: + + 1) An IOV is being appended to an existing Tag with an existing Payload. + In this case, we just take all data from the command line. + + 2) No metadata file is given, so we assume that ALL upload metadata is coming from the command line. + + 3) A metadata file is given, hence we parse the file, and then iterate through command line arguments + since these override the options set in the metadata file. + + """ + + # Hash to use, entirely from command line + if command_line_data.hashToUse != None: + command_line_data.userText = "" + metadata_dictionary = command_line_data.__dict__ + elif command_line_data.metadataFile == None: + if command_line_data.sourceDB != None and (command_line_data.inputTag == None or command_line_data.destinationTag == None or command_line_data.destinationDatabase == None): + basepath = command_line_data.sourceDB.rsplit('.db', 1)[0].rsplit('.txt', 1)[0] + basename = os.path.basename(basepath) + dataFilename = '%s.db' % basepath + metadataFilename = '%s.txt' % basepath + # Data file + try: + with open(dataFilename, 'rb') as dataFile: + pass + except IOError as e: + errMsg = 'Impossible to open SQLite data file %s' %dataFilename + print( errMsg ) + ret['status'] = -3 + ret['error'] = errMsg + return ret + + # Metadata file + + try: + with open(metadataFilename, 'rb') as metadataFile: + pass + except IOError as e: + if e.errno != errno.ENOENT: + errMsg = 'Impossible to open file %s (for other reason than not existing)' %metadataFilename + ret = {} + ret['status'] = -4 + ret['error'] = errMsg + exit (ret) + + if getInput('y', '\nIt looks like the metadata file %s does not exist and not enough parameters were received in the command line. Do you want me to create it and help you fill it?\nAnswer [y]: ' % metadataFilename).lower() != 'y': + errMsg = 'Metadata file %s does not exist' %metadataFilename + ret = {} + ret['status'] = -5 + ret['error'] = errMsg + exit(ret) + # Wizard + runWizard(basename, dataFilename, metadataFilename) + command_line_data.metadataFile = metadataFilename + else: + command_line_data.userText = command_line_data.userText\ + if command_line_data.userText != None\ + else str(raw_input("Tag's description [can be empty]:")) + metadata_dictionary = command_line_data.__dict__ + + if command_line_data.metadataFile != None: + metadata_dictionary = json.loads("".join(open(os.path.abspath(command_line_data.metadataFile), "r").readlines())) + metadata_dictionary["username"] = username + metadata_dictionary["password"] = password + metadata_dictionary["userText"] = metadata_dictionary.get("userText")\ + if metadata_dictionary.get("userText") != None\ + else str(raw_input("Tag's description [can be empty]:")) + # set the server to use to be the default one + metadata_dictionary["server"] = server_alias_to_url[None] + + # go through command line options and, if they are set, overwrite entries + for (option_name, option_value) in command_line_data.__dict__.items(): + # if the metadata_dictionary sets this, overwrite it + if option_name != "destinationTags": + if option_value != None or (option_value == None and not(option_name in metadata_dictionary.keys())): + # if option_value has a value, override the metadata file entry + # or if option_value is None but the metadata file doesn't give a value, + # set the entry to None as well + metadata_dictionary[option_name] = option_value + else: + if option_value != {None:{}}: + metadata_dictionary["destinationTags"] = {option_value:{}} + elif option_value == {None:{}} and not("destinationTags" in metadata_dictionary.keys()): + metadata_dictionary["destinationTags"] = {None:{}} + + if command_line_data.review_options: + defaults = { + "since" : "Since of first IOV", + "userText" : "Populated by upload process", + "netrc" : "None given", + "fcsr_filter" : "Don't apply", + "hashToUse" : "Using local SQLite file instead" + } + print("Configuration to use for the upload:") + for key in metadata_dictionary: + if not(key) in ["username", "password", "destinationTag"]: + value_to_print = metadata_dictionary[key] if metadata_dictionary[key] != None else defaults[key] + print("\t%s : %s" % (key, value_to_print)) + + if raw_input("\nDo you want to continue? [y/n] ") != "y": + exit() + + return metadata_dictionary + +def get_version(url): + return requests.get(url + "script_version/", verify=False) + + +if __name__ == "__main__": + + upload_metadata = parse_arguments() + + # upload_metadata should be used to decide the service url + final_service_url = upload_metadata["server"] + try: + response = get_version(final_service_url) + server_version = response.json() + except Exception as e: + print(horizontal_rule) + print(e) + print("Could not connect to server at %s"%final_service_url) + print("If you specified a server please check it is correct. If that is not the issue please contact the AlcaDB team.") + print(horizontal_rule) + exit(1) + + if server_version["version"] != __version__: + print(horizontal_rule) + print("Local upload script is different than server version. Please run the following command to get the latest script.") + print("curl --insecure -o uploadConditions.py %sget_upload_script/ && chmod +x uploadConditions.py;"%final_service_url) + print(horizontal_rule) + exit(1) + + import CondCore.Utilities.CondDBFW.data_sources as data_sources + + upload_metadata["sqlite_file"] = upload_metadata.get("sourceDB") + + try: + os.mkdir('upload_logs') + except OSError as e: + pass + + # make new dictionary, and copy over everything except "metadata_source" + upload_metadata_argument = {} + for (key, value) in upload_metadata.items(): + if key != "metadata_source": + upload_metadata_argument[key] = value + + upload_metadata["metadata_source"] = data_sources.json_data_node.make(upload_metadata_argument) + try: + # pass dictionary as arguments to match keywords - the constructor has a **kwargs parameter to deal with stray arguments + run_upload(**upload_metadata) + print(horizontal_rule) + print("Process completed without issues. Please check logs for further details.") + print(horizontal_rule) + except SystemExit as e: + print(horizontal_rule) + print("Process exited abnormally. Please check logs for details.") + print(horizontal_rule) + exit(1) + exit(0) From 802f7c66a914f9cf42a4d4b28a579dcb606c233e Mon Sep 17 00:00:00 2001 From: Andres Date: Tue, 20 Apr 2021 16:57:03 +0200 Subject: [PATCH 2/5] Revert commit --- .../Utilities/scripts/uploadConditions_v2.py | 470 ------------------ 1 file changed, 470 deletions(-) delete mode 100755 CondCore/Utilities/scripts/uploadConditions_v2.py diff --git a/CondCore/Utilities/scripts/uploadConditions_v2.py b/CondCore/Utilities/scripts/uploadConditions_v2.py deleted file mode 100755 index 55c8f36e98139..0000000000000 --- a/CondCore/Utilities/scripts/uploadConditions_v2.py +++ /dev/null @@ -1,470 +0,0 @@ -#!/usr/bin/env python -""" - -Joshua Dawes - CERN, CMS - The University of Manchester - -Upload script wrapper - controls the automatic update system. - -Note: the name of the file follows a different convention to the others because it should be the same as the current upload script name. - -Takes user arguments and passes them to the main upload module CondDBFW.uploads, once the correct version exists. - -1. Ask the server corresponding to the database we're uploading to which version of CondDBFW it has (query the /conddbfw_version/ url). -2. Decide which directory that we can write to - either the current local directory, or /tmp/random_string/. -3. Pull the commit returned from the server into the directory from step 2. -4. Invoke the CondDBFW.uploads module with the arguments given to this script. - -""" - -__version__ = 1 - -#import pycurl -import requests -import urllib3 -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) -try: - from StringIO import StringIO -except: - pass -import traceback -import sys -import os -import json -import subprocess -import argparse -import netrc -import shutil -import getpass -import errno -import sqlite3 - - -horizontal_rule = "="*60 - -def run_upload(**parameters): - """ - Imports CondDBFW.uploads and runs the upload with the upload metadata obtained. - """ - try: - import CondCore.Utilities.CondDBFW.uploads as uploads - except Exception as e: - traceback.print_exc() - exit("CondDBFW or one of its dependencies could not be imported.\n"\ - + "If the CondDBFW directory exists, you are likely not in a CMSSW environment.") - # we have CondDBFW, so just call the module with the parameters given in the command line - uploader = uploads.uploader(**parameters) - result = uploader.upload() - -def getInput(default, prompt = ''): - '''Like raw_input() but with a default and automatic strip(). - ''' - - answer = raw_input(prompt) - if answer: - return answer.strip() - - return default.strip() - - -def getInputWorkflow(prompt = ''): - '''Like getInput() but tailored to get target workflows (synchronization options). - ''' - - while True: - workflow = getInput(defaultWorkflow, prompt) - - if workflow in frozenset(['offline', 'hlt', 'express', 'prompt', 'pcl']): - return workflow - - print('Please specify one of the allowed workflows. See above for the explanation on each of them.') - - -def getInputChoose(optionsList, default, prompt = ''): - '''Makes the user choose from a list of options. - ''' - - while True: - index = getInput(default, prompt) - - try: - return optionsList[int(index)] - except ValueError: - print('Please specify an index of the list (i.e. integer).') - except IndexError: - print('The index you provided is not in the given list.') - - -def getInputRepeat(prompt = ''): - '''Like raw_input() but repeats if nothing is provided and automatic strip(). - ''' - - while True: - answer = raw_input(prompt) - if answer: - return answer.strip() - - print('You need to provide a value.') - -def runWizard(basename, dataFilename, metadataFilename): - while True: - print('''\nWizard for metadata for %s - -I will ask you some questions to fill the metadata file. For some of the questions there are defaults between square brackets (i.e. []), leave empty (i.e. hit Enter) to use them.''' % basename) - - # Try to get the available inputTags - try: - dataConnection = sqlite3.connect(dataFilename) - dataCursor = dataConnection.cursor() - dataCursor.execute('select name from sqlite_master where type == "table"') - tables = set(zip(*dataCursor.fetchall())[0]) - - # only conddb V2 supported... - if 'TAG' in tables: - dataCursor.execute('select NAME from TAG') - # In any other case, do not try to get the inputTags - else: - raise Exception() - - inputTags = dataCursor.fetchall() - if len(inputTags) == 0: - raise Exception() - inputTags = list(zip(*inputTags))[0] - - except Exception: - inputTags = [] - - if len(inputTags) == 0: - print('\nI could not find any input tag in your data file, but you can still specify one manually.') - - inputTag = getInputRepeat( - '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. BeamSpotObject_ByRun\ninputTag: ') - - else: - print('\nI found the following input tags in your SQLite data file:') - for (index, inputTag) in enumerate(inputTags): - print(' %s) %s' % (index, inputTag)) - - inputTag = getInputChoose(inputTags, '0', - '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. 0 (you select the first in the list)\ninputTag [0]: ') - - destinationDatabase = '' - ntry = 0 - while ( destinationDatabase != 'oracle://cms_orcon_prod/CMS_CONDITIONS' and destinationDatabase != 'oracle://cms_orcoff_prep/CMS_CONDITIONS' ): - if ntry==0: - inputMessage = \ - '\nWhich is the destination database where the tags should be exported? \nPossible choices: oracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \ndestinationDatabase: ' - elif ntry==1: - inputMessage = \ - '\nPlease choose one of the two valid destinations: \noracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \ -\ndestinationDatabase: ' - else: - raise Exception('No valid destination chosen. Bailing out...') - destinationDatabase = getInputRepeat(inputMessage) - ntry += 1 - - while True: - since = getInput('', - '\nWhich is the given since? (if not specified, the one from the SQLite data file will be taken -- note that even if specified, still this may not be the final since, depending on the synchronization options you select later: if the synchronization target is not offline, and the since you give is smaller than the next possible one (i.e. you give a run number earlier than the one which will be started/processed next in prompt/hlt/express), the DropBox will move the since ahead to go to the first safe run instead of the value you gave)\ne.g. 1234\nsince []: ') - if not since: - since = None - break - else: - try: - since = int(since) - break - except ValueError: - print('The since value has to be an integer or empty (null).') - - userText = getInput('', - '\nWrite any comments/text you may want to describe your request\ne.g. Muon alignment scenario for...\nuserText []: ') - - destinationTags = {} - while True: - destinationTag = getInput('', - '\nWhich is the next destination tag to be added (leave empty to stop)?\ne.g. BeamSpotObjects_PCL_byRun_v0_offline\ndestinationTag []: ') - if not destinationTag: - if len(destinationTags) == 0: - print('There must be at least one destination tag.') - continue - break - - if destinationTag in destinationTags: - print( - 'You already added this destination tag. Overwriting the previous one with this new one.') - - destinationTags[destinationTag] = { - } - - metadata = { - 'destinationDatabase': destinationDatabase, - 'destinationTags': destinationTags, - 'inputTag': inputTag, - 'since': since, - 'userText': userText, - } - - metadata = json.dumps(metadata, sort_keys=True, indent=4) - print('\nThis is the generated metadata:\n%s' % metadata) - - if getInput('n', - '\nIs it fine (i.e. save in %s and *upload* the conditions if this is the latest file)?\nAnswer [n]: ' % metadataFilename).lower() == 'y': - break - print('Saving generated metadata in %s...', metadataFilename) - with open(metadataFilename, 'wb') as metadataFile: - metadataFile.write(metadata) - -def parse_arguments(): - # read in command line arguments, and build metadata dictionary from them - parser = argparse.ArgumentParser(prog="cmsDbUpload client", description="CMS Conditions Upload Script in CondDBFW.") - - parser.add_argument("--sourceDB", type=str, help="DB to find Tags, IOVs + Payloads in.", required=False) - - # metadata arguments - parser.add_argument("--inputTag", type=str,\ - help="Tag to take IOVs + Payloads from in --sourceDB.", required=False) - parser.add_argument("--destinationTag", type=str,\ - help="Tag to copy IOVs + Payloads to in --destDB.", required=False) - parser.add_argument("--destinationDatabase", type=str,\ - help="Database to copy IOVs + Payloads to.", required=False) - parser.add_argument("--since", type=int,\ - help="Since to take IOVs from.", required=False) - parser.add_argument("--userText", type=str,\ - help="Description of --destTag (can be empty).") - - # non-metadata arguments - parser.add_argument("--metadataFile", "-m", type=str, help="Metadata file to take metadata from.", required=False) - - parser.add_argument("--debug", required=False, action="store_true") - parser.add_argument("--verbose", required=False, action="store_true") - parser.add_argument("--testing", required=False, action="store_true") - parser.add_argument("--fcsr-filter", type=str, help="Synchronization to take FCSR from for local filtering of IOVs.", required=False) - - parser.add_argument("--netrc", required=False) - - parser.add_argument("--hashToUse", required=False) - - parser.add_argument("--server", required=False) - - parser.add_argument("--review-options", required=False, action="store_true") - - parser.add_argument("--replay-file", required=False) - - command_line_data = parser.parse_args() - - if command_line_data.replay_file: - dictionary = json.loads("".join(open(command_line_data.replay_file, "r").readlines())) - command_line_data.tier0_response = dictionary["tier0_response"] - - # default is the production server, which can point to either database anyway - server_alias_to_url = { - "prep" : "https://cms-conddb-dev.cern.ch/cmsDbCondUpload/", - "dev" : "https://cms-conddb-dev.cern.ch/cmsDbCondUpload/", - "prod" : "https://cms-conddb.cern.ch/cmsDbCondUpload/", - None : "https://cms-conddb.cern.ch/cmsDbCondUpload/" - } - - # if prep, prod or None were given, convert to URLs in dictionary server_alias_to_url - # if not, assume a URL has been given and use this instead - if command_line_data.server in server_alias_to_url.keys(): - command_line_data.server = server_alias_to_url[command_line_data.server] - - # resolve destination databases - database_alias_to_connection = { - "prep": "oracle://cms_orcoff_prep/CMS_CONDITIONS", - "dev": "oracle://cms_orcoff_prep/CMS_CONDITIONS", - "prod": "oracle://cms_orcon_adg/CMS_CONDITIONS" - } - - if command_line_data.destinationDatabase in database_alias_to_connection.keys(): - command_line_data.destinationDatabase = database_alias_to_connection[command_line_data.destinationDatabase] - - # use netrc to get username and password - try: - netrc_file = command_line_data.netrc - netrc_authenticators = netrc.netrc(netrc_file).authenticators("ConditionUploader") - if netrc_authenticators == None: - print("Your netrc file must contain the key 'ConditionUploader'.") - manual_input = raw_input("Do you want to try to type your credentials? ") - if manual_input == "y": - # ask for username and password - username = raw_input("Username: ") - password = getpass.getpass("Password: ") - else: - exit() - else: - print("Read your credentials from ~/.netrc. If you want to use a different file, supply its name with the --netrc argument.") - username = netrc_authenticators[0] - password = netrc_authenticators[2] - except: - print("Couldn't obtain your credentials (either from netrc or manual input).") - exit() - - command_line_data.username = username - command_line_data.password = password - # this will be used as the final destinationTags value by all input methods - # apart from the metadata file - command_line_data.destinationTags = {command_line_data.destinationTag:{}} - - """ - Construct metadata_dictionary: - Currently, this is 3 cases: - - 1) An IOV is being appended to an existing Tag with an existing Payload. - In this case, we just take all data from the command line. - - 2) No metadata file is given, so we assume that ALL upload metadata is coming from the command line. - - 3) A metadata file is given, hence we parse the file, and then iterate through command line arguments - since these override the options set in the metadata file. - - """ - - # Hash to use, entirely from command line - if command_line_data.hashToUse != None: - command_line_data.userText = "" - metadata_dictionary = command_line_data.__dict__ - elif command_line_data.metadataFile == None: - if command_line_data.sourceDB != None and (command_line_data.inputTag == None or command_line_data.destinationTag == None or command_line_data.destinationDatabase == None): - basepath = command_line_data.sourceDB.rsplit('.db', 1)[0].rsplit('.txt', 1)[0] - basename = os.path.basename(basepath) - dataFilename = '%s.db' % basepath - metadataFilename = '%s.txt' % basepath - # Data file - try: - with open(dataFilename, 'rb') as dataFile: - pass - except IOError as e: - errMsg = 'Impossible to open SQLite data file %s' %dataFilename - print( errMsg ) - ret['status'] = -3 - ret['error'] = errMsg - return ret - - # Metadata file - - try: - with open(metadataFilename, 'rb') as metadataFile: - pass - except IOError as e: - if e.errno != errno.ENOENT: - errMsg = 'Impossible to open file %s (for other reason than not existing)' %metadataFilename - ret = {} - ret['status'] = -4 - ret['error'] = errMsg - exit (ret) - - if getInput('y', '\nIt looks like the metadata file %s does not exist and not enough parameters were received in the command line. Do you want me to create it and help you fill it?\nAnswer [y]: ' % metadataFilename).lower() != 'y': - errMsg = 'Metadata file %s does not exist' %metadataFilename - ret = {} - ret['status'] = -5 - ret['error'] = errMsg - exit(ret) - # Wizard - runWizard(basename, dataFilename, metadataFilename) - command_line_data.metadataFile = metadataFilename - else: - command_line_data.userText = command_line_data.userText\ - if command_line_data.userText != None\ - else str(raw_input("Tag's description [can be empty]:")) - metadata_dictionary = command_line_data.__dict__ - - if command_line_data.metadataFile != None: - metadata_dictionary = json.loads("".join(open(os.path.abspath(command_line_data.metadataFile), "r").readlines())) - metadata_dictionary["username"] = username - metadata_dictionary["password"] = password - metadata_dictionary["userText"] = metadata_dictionary.get("userText")\ - if metadata_dictionary.get("userText") != None\ - else str(raw_input("Tag's description [can be empty]:")) - # set the server to use to be the default one - metadata_dictionary["server"] = server_alias_to_url[None] - - # go through command line options and, if they are set, overwrite entries - for (option_name, option_value) in command_line_data.__dict__.items(): - # if the metadata_dictionary sets this, overwrite it - if option_name != "destinationTags": - if option_value != None or (option_value == None and not(option_name in metadata_dictionary.keys())): - # if option_value has a value, override the metadata file entry - # or if option_value is None but the metadata file doesn't give a value, - # set the entry to None as well - metadata_dictionary[option_name] = option_value - else: - if option_value != {None:{}}: - metadata_dictionary["destinationTags"] = {option_value:{}} - elif option_value == {None:{}} and not("destinationTags" in metadata_dictionary.keys()): - metadata_dictionary["destinationTags"] = {None:{}} - - if command_line_data.review_options: - defaults = { - "since" : "Since of first IOV", - "userText" : "Populated by upload process", - "netrc" : "None given", - "fcsr_filter" : "Don't apply", - "hashToUse" : "Using local SQLite file instead" - } - print("Configuration to use for the upload:") - for key in metadata_dictionary: - if not(key) in ["username", "password", "destinationTag"]: - value_to_print = metadata_dictionary[key] if metadata_dictionary[key] != None else defaults[key] - print("\t%s : %s" % (key, value_to_print)) - - if raw_input("\nDo you want to continue? [y/n] ") != "y": - exit() - - return metadata_dictionary - -def get_version(url): - return requests.get(url + "script_version/", verify=False) - - -if __name__ == "__main__": - - upload_metadata = parse_arguments() - - # upload_metadata should be used to decide the service url - final_service_url = upload_metadata["server"] - try: - response = get_version(final_service_url) - server_version = response.json() - except Exception as e: - print(horizontal_rule) - print(e) - print("Could not connect to server at %s"%final_service_url) - print("If you specified a server please check it is correct. If that is not the issue please contact the AlcaDB team.") - print(horizontal_rule) - exit(1) - - if server_version["version"] != __version__: - print(horizontal_rule) - print("Local upload script is different than server version. Please run the following command to get the latest script.") - print("curl --insecure -o uploadConditions.py %sget_upload_script/ && chmod +x uploadConditions.py;"%final_service_url) - print(horizontal_rule) - exit(1) - - import CondCore.Utilities.CondDBFW.data_sources as data_sources - - upload_metadata["sqlite_file"] = upload_metadata.get("sourceDB") - - try: - os.mkdir('upload_logs') - except OSError as e: - pass - - # make new dictionary, and copy over everything except "metadata_source" - upload_metadata_argument = {} - for (key, value) in upload_metadata.items(): - if key != "metadata_source": - upload_metadata_argument[key] = value - - upload_metadata["metadata_source"] = data_sources.json_data_node.make(upload_metadata_argument) - try: - # pass dictionary as arguments to match keywords - the constructor has a **kwargs parameter to deal with stray arguments - run_upload(**upload_metadata) - print(horizontal_rule) - print("Process completed without issues. Please check logs for further details.") - print(horizontal_rule) - except SystemExit as e: - print(horizontal_rule) - print("Process exited abnormally. Please check logs for details.") - print(horizontal_rule) - exit(1) - exit(0) From c2c5864734306555f659d5800275b09c468b710a Mon Sep 17 00:00:00 2001 From: Andres Date: Tue, 20 Apr 2021 16:59:28 +0200 Subject: [PATCH 3/5] Add new uploader script --- .../Utilities/scripts/uploadConditions_v2.py | 470 ++++++++++++++++++ 1 file changed, 470 insertions(+) create mode 100755 CondCore/Utilities/scripts/uploadConditions_v2.py diff --git a/CondCore/Utilities/scripts/uploadConditions_v2.py b/CondCore/Utilities/scripts/uploadConditions_v2.py new file mode 100755 index 0000000000000..55c8f36e98139 --- /dev/null +++ b/CondCore/Utilities/scripts/uploadConditions_v2.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python +""" + +Joshua Dawes - CERN, CMS - The University of Manchester + +Upload script wrapper - controls the automatic update system. + +Note: the name of the file follows a different convention to the others because it should be the same as the current upload script name. + +Takes user arguments and passes them to the main upload module CondDBFW.uploads, once the correct version exists. + +1. Ask the server corresponding to the database we're uploading to which version of CondDBFW it has (query the /conddbfw_version/ url). +2. Decide which directory that we can write to - either the current local directory, or /tmp/random_string/. +3. Pull the commit returned from the server into the directory from step 2. +4. Invoke the CondDBFW.uploads module with the arguments given to this script. + +""" + +__version__ = 1 + +#import pycurl +import requests +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +try: + from StringIO import StringIO +except: + pass +import traceback +import sys +import os +import json +import subprocess +import argparse +import netrc +import shutil +import getpass +import errno +import sqlite3 + + +horizontal_rule = "="*60 + +def run_upload(**parameters): + """ + Imports CondDBFW.uploads and runs the upload with the upload metadata obtained. + """ + try: + import CondCore.Utilities.CondDBFW.uploads as uploads + except Exception as e: + traceback.print_exc() + exit("CondDBFW or one of its dependencies could not be imported.\n"\ + + "If the CondDBFW directory exists, you are likely not in a CMSSW environment.") + # we have CondDBFW, so just call the module with the parameters given in the command line + uploader = uploads.uploader(**parameters) + result = uploader.upload() + +def getInput(default, prompt = ''): + '''Like raw_input() but with a default and automatic strip(). + ''' + + answer = raw_input(prompt) + if answer: + return answer.strip() + + return default.strip() + + +def getInputWorkflow(prompt = ''): + '''Like getInput() but tailored to get target workflows (synchronization options). + ''' + + while True: + workflow = getInput(defaultWorkflow, prompt) + + if workflow in frozenset(['offline', 'hlt', 'express', 'prompt', 'pcl']): + return workflow + + print('Please specify one of the allowed workflows. See above for the explanation on each of them.') + + +def getInputChoose(optionsList, default, prompt = ''): + '''Makes the user choose from a list of options. + ''' + + while True: + index = getInput(default, prompt) + + try: + return optionsList[int(index)] + except ValueError: + print('Please specify an index of the list (i.e. integer).') + except IndexError: + print('The index you provided is not in the given list.') + + +def getInputRepeat(prompt = ''): + '''Like raw_input() but repeats if nothing is provided and automatic strip(). + ''' + + while True: + answer = raw_input(prompt) + if answer: + return answer.strip() + + print('You need to provide a value.') + +def runWizard(basename, dataFilename, metadataFilename): + while True: + print('''\nWizard for metadata for %s + +I will ask you some questions to fill the metadata file. For some of the questions there are defaults between square brackets (i.e. []), leave empty (i.e. hit Enter) to use them.''' % basename) + + # Try to get the available inputTags + try: + dataConnection = sqlite3.connect(dataFilename) + dataCursor = dataConnection.cursor() + dataCursor.execute('select name from sqlite_master where type == "table"') + tables = set(zip(*dataCursor.fetchall())[0]) + + # only conddb V2 supported... + if 'TAG' in tables: + dataCursor.execute('select NAME from TAG') + # In any other case, do not try to get the inputTags + else: + raise Exception() + + inputTags = dataCursor.fetchall() + if len(inputTags) == 0: + raise Exception() + inputTags = list(zip(*inputTags))[0] + + except Exception: + inputTags = [] + + if len(inputTags) == 0: + print('\nI could not find any input tag in your data file, but you can still specify one manually.') + + inputTag = getInputRepeat( + '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. BeamSpotObject_ByRun\ninputTag: ') + + else: + print('\nI found the following input tags in your SQLite data file:') + for (index, inputTag) in enumerate(inputTags): + print(' %s) %s' % (index, inputTag)) + + inputTag = getInputChoose(inputTags, '0', + '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. 0 (you select the first in the list)\ninputTag [0]: ') + + destinationDatabase = '' + ntry = 0 + while ( destinationDatabase != 'oracle://cms_orcon_prod/CMS_CONDITIONS' and destinationDatabase != 'oracle://cms_orcoff_prep/CMS_CONDITIONS' ): + if ntry==0: + inputMessage = \ + '\nWhich is the destination database where the tags should be exported? \nPossible choices: oracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \ndestinationDatabase: ' + elif ntry==1: + inputMessage = \ + '\nPlease choose one of the two valid destinations: \noracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \ +\ndestinationDatabase: ' + else: + raise Exception('No valid destination chosen. Bailing out...') + destinationDatabase = getInputRepeat(inputMessage) + ntry += 1 + + while True: + since = getInput('', + '\nWhich is the given since? (if not specified, the one from the SQLite data file will be taken -- note that even if specified, still this may not be the final since, depending on the synchronization options you select later: if the synchronization target is not offline, and the since you give is smaller than the next possible one (i.e. you give a run number earlier than the one which will be started/processed next in prompt/hlt/express), the DropBox will move the since ahead to go to the first safe run instead of the value you gave)\ne.g. 1234\nsince []: ') + if not since: + since = None + break + else: + try: + since = int(since) + break + except ValueError: + print('The since value has to be an integer or empty (null).') + + userText = getInput('', + '\nWrite any comments/text you may want to describe your request\ne.g. Muon alignment scenario for...\nuserText []: ') + + destinationTags = {} + while True: + destinationTag = getInput('', + '\nWhich is the next destination tag to be added (leave empty to stop)?\ne.g. BeamSpotObjects_PCL_byRun_v0_offline\ndestinationTag []: ') + if not destinationTag: + if len(destinationTags) == 0: + print('There must be at least one destination tag.') + continue + break + + if destinationTag in destinationTags: + print( + 'You already added this destination tag. Overwriting the previous one with this new one.') + + destinationTags[destinationTag] = { + } + + metadata = { + 'destinationDatabase': destinationDatabase, + 'destinationTags': destinationTags, + 'inputTag': inputTag, + 'since': since, + 'userText': userText, + } + + metadata = json.dumps(metadata, sort_keys=True, indent=4) + print('\nThis is the generated metadata:\n%s' % metadata) + + if getInput('n', + '\nIs it fine (i.e. save in %s and *upload* the conditions if this is the latest file)?\nAnswer [n]: ' % metadataFilename).lower() == 'y': + break + print('Saving generated metadata in %s...', metadataFilename) + with open(metadataFilename, 'wb') as metadataFile: + metadataFile.write(metadata) + +def parse_arguments(): + # read in command line arguments, and build metadata dictionary from them + parser = argparse.ArgumentParser(prog="cmsDbUpload client", description="CMS Conditions Upload Script in CondDBFW.") + + parser.add_argument("--sourceDB", type=str, help="DB to find Tags, IOVs + Payloads in.", required=False) + + # metadata arguments + parser.add_argument("--inputTag", type=str,\ + help="Tag to take IOVs + Payloads from in --sourceDB.", required=False) + parser.add_argument("--destinationTag", type=str,\ + help="Tag to copy IOVs + Payloads to in --destDB.", required=False) + parser.add_argument("--destinationDatabase", type=str,\ + help="Database to copy IOVs + Payloads to.", required=False) + parser.add_argument("--since", type=int,\ + help="Since to take IOVs from.", required=False) + parser.add_argument("--userText", type=str,\ + help="Description of --destTag (can be empty).") + + # non-metadata arguments + parser.add_argument("--metadataFile", "-m", type=str, help="Metadata file to take metadata from.", required=False) + + parser.add_argument("--debug", required=False, action="store_true") + parser.add_argument("--verbose", required=False, action="store_true") + parser.add_argument("--testing", required=False, action="store_true") + parser.add_argument("--fcsr-filter", type=str, help="Synchronization to take FCSR from for local filtering of IOVs.", required=False) + + parser.add_argument("--netrc", required=False) + + parser.add_argument("--hashToUse", required=False) + + parser.add_argument("--server", required=False) + + parser.add_argument("--review-options", required=False, action="store_true") + + parser.add_argument("--replay-file", required=False) + + command_line_data = parser.parse_args() + + if command_line_data.replay_file: + dictionary = json.loads("".join(open(command_line_data.replay_file, "r").readlines())) + command_line_data.tier0_response = dictionary["tier0_response"] + + # default is the production server, which can point to either database anyway + server_alias_to_url = { + "prep" : "https://cms-conddb-dev.cern.ch/cmsDbCondUpload/", + "dev" : "https://cms-conddb-dev.cern.ch/cmsDbCondUpload/", + "prod" : "https://cms-conddb.cern.ch/cmsDbCondUpload/", + None : "https://cms-conddb.cern.ch/cmsDbCondUpload/" + } + + # if prep, prod or None were given, convert to URLs in dictionary server_alias_to_url + # if not, assume a URL has been given and use this instead + if command_line_data.server in server_alias_to_url.keys(): + command_line_data.server = server_alias_to_url[command_line_data.server] + + # resolve destination databases + database_alias_to_connection = { + "prep": "oracle://cms_orcoff_prep/CMS_CONDITIONS", + "dev": "oracle://cms_orcoff_prep/CMS_CONDITIONS", + "prod": "oracle://cms_orcon_adg/CMS_CONDITIONS" + } + + if command_line_data.destinationDatabase in database_alias_to_connection.keys(): + command_line_data.destinationDatabase = database_alias_to_connection[command_line_data.destinationDatabase] + + # use netrc to get username and password + try: + netrc_file = command_line_data.netrc + netrc_authenticators = netrc.netrc(netrc_file).authenticators("ConditionUploader") + if netrc_authenticators == None: + print("Your netrc file must contain the key 'ConditionUploader'.") + manual_input = raw_input("Do you want to try to type your credentials? ") + if manual_input == "y": + # ask for username and password + username = raw_input("Username: ") + password = getpass.getpass("Password: ") + else: + exit() + else: + print("Read your credentials from ~/.netrc. If you want to use a different file, supply its name with the --netrc argument.") + username = netrc_authenticators[0] + password = netrc_authenticators[2] + except: + print("Couldn't obtain your credentials (either from netrc or manual input).") + exit() + + command_line_data.username = username + command_line_data.password = password + # this will be used as the final destinationTags value by all input methods + # apart from the metadata file + command_line_data.destinationTags = {command_line_data.destinationTag:{}} + + """ + Construct metadata_dictionary: + Currently, this is 3 cases: + + 1) An IOV is being appended to an existing Tag with an existing Payload. + In this case, we just take all data from the command line. + + 2) No metadata file is given, so we assume that ALL upload metadata is coming from the command line. + + 3) A metadata file is given, hence we parse the file, and then iterate through command line arguments + since these override the options set in the metadata file. + + """ + + # Hash to use, entirely from command line + if command_line_data.hashToUse != None: + command_line_data.userText = "" + metadata_dictionary = command_line_data.__dict__ + elif command_line_data.metadataFile == None: + if command_line_data.sourceDB != None and (command_line_data.inputTag == None or command_line_data.destinationTag == None or command_line_data.destinationDatabase == None): + basepath = command_line_data.sourceDB.rsplit('.db', 1)[0].rsplit('.txt', 1)[0] + basename = os.path.basename(basepath) + dataFilename = '%s.db' % basepath + metadataFilename = '%s.txt' % basepath + # Data file + try: + with open(dataFilename, 'rb') as dataFile: + pass + except IOError as e: + errMsg = 'Impossible to open SQLite data file %s' %dataFilename + print( errMsg ) + ret['status'] = -3 + ret['error'] = errMsg + return ret + + # Metadata file + + try: + with open(metadataFilename, 'rb') as metadataFile: + pass + except IOError as e: + if e.errno != errno.ENOENT: + errMsg = 'Impossible to open file %s (for other reason than not existing)' %metadataFilename + ret = {} + ret['status'] = -4 + ret['error'] = errMsg + exit (ret) + + if getInput('y', '\nIt looks like the metadata file %s does not exist and not enough parameters were received in the command line. Do you want me to create it and help you fill it?\nAnswer [y]: ' % metadataFilename).lower() != 'y': + errMsg = 'Metadata file %s does not exist' %metadataFilename + ret = {} + ret['status'] = -5 + ret['error'] = errMsg + exit(ret) + # Wizard + runWizard(basename, dataFilename, metadataFilename) + command_line_data.metadataFile = metadataFilename + else: + command_line_data.userText = command_line_data.userText\ + if command_line_data.userText != None\ + else str(raw_input("Tag's description [can be empty]:")) + metadata_dictionary = command_line_data.__dict__ + + if command_line_data.metadataFile != None: + metadata_dictionary = json.loads("".join(open(os.path.abspath(command_line_data.metadataFile), "r").readlines())) + metadata_dictionary["username"] = username + metadata_dictionary["password"] = password + metadata_dictionary["userText"] = metadata_dictionary.get("userText")\ + if metadata_dictionary.get("userText") != None\ + else str(raw_input("Tag's description [can be empty]:")) + # set the server to use to be the default one + metadata_dictionary["server"] = server_alias_to_url[None] + + # go through command line options and, if they are set, overwrite entries + for (option_name, option_value) in command_line_data.__dict__.items(): + # if the metadata_dictionary sets this, overwrite it + if option_name != "destinationTags": + if option_value != None or (option_value == None and not(option_name in metadata_dictionary.keys())): + # if option_value has a value, override the metadata file entry + # or if option_value is None but the metadata file doesn't give a value, + # set the entry to None as well + metadata_dictionary[option_name] = option_value + else: + if option_value != {None:{}}: + metadata_dictionary["destinationTags"] = {option_value:{}} + elif option_value == {None:{}} and not("destinationTags" in metadata_dictionary.keys()): + metadata_dictionary["destinationTags"] = {None:{}} + + if command_line_data.review_options: + defaults = { + "since" : "Since of first IOV", + "userText" : "Populated by upload process", + "netrc" : "None given", + "fcsr_filter" : "Don't apply", + "hashToUse" : "Using local SQLite file instead" + } + print("Configuration to use for the upload:") + for key in metadata_dictionary: + if not(key) in ["username", "password", "destinationTag"]: + value_to_print = metadata_dictionary[key] if metadata_dictionary[key] != None else defaults[key] + print("\t%s : %s" % (key, value_to_print)) + + if raw_input("\nDo you want to continue? [y/n] ") != "y": + exit() + + return metadata_dictionary + +def get_version(url): + return requests.get(url + "script_version/", verify=False) + + +if __name__ == "__main__": + + upload_metadata = parse_arguments() + + # upload_metadata should be used to decide the service url + final_service_url = upload_metadata["server"] + try: + response = get_version(final_service_url) + server_version = response.json() + except Exception as e: + print(horizontal_rule) + print(e) + print("Could not connect to server at %s"%final_service_url) + print("If you specified a server please check it is correct. If that is not the issue please contact the AlcaDB team.") + print(horizontal_rule) + exit(1) + + if server_version["version"] != __version__: + print(horizontal_rule) + print("Local upload script is different than server version. Please run the following command to get the latest script.") + print("curl --insecure -o uploadConditions.py %sget_upload_script/ && chmod +x uploadConditions.py;"%final_service_url) + print(horizontal_rule) + exit(1) + + import CondCore.Utilities.CondDBFW.data_sources as data_sources + + upload_metadata["sqlite_file"] = upload_metadata.get("sourceDB") + + try: + os.mkdir('upload_logs') + except OSError as e: + pass + + # make new dictionary, and copy over everything except "metadata_source" + upload_metadata_argument = {} + for (key, value) in upload_metadata.items(): + if key != "metadata_source": + upload_metadata_argument[key] = value + + upload_metadata["metadata_source"] = data_sources.json_data_node.make(upload_metadata_argument) + try: + # pass dictionary as arguments to match keywords - the constructor has a **kwargs parameter to deal with stray arguments + run_upload(**upload_metadata) + print(horizontal_rule) + print("Process completed without issues. Please check logs for further details.") + print(horizontal_rule) + except SystemExit as e: + print(horizontal_rule) + print("Process exited abnormally. Please check logs for details.") + print(horizontal_rule) + exit(1) + exit(0) From f9f3ef9b88dc54d0e8d91bbed41f11c0d4e58585 Mon Sep 17 00:00:00 2001 From: Andres Date: Tue, 20 Apr 2021 17:16:38 +0200 Subject: [PATCH 4/5] Update script header --- CondCore/Utilities/scripts/uploadConditions_v2.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CondCore/Utilities/scripts/uploadConditions_v2.py b/CondCore/Utilities/scripts/uploadConditions_v2.py index 55c8f36e98139..03e84d6866210 100755 --- a/CondCore/Utilities/scripts/uploadConditions_v2.py +++ b/CondCore/Utilities/scripts/uploadConditions_v2.py @@ -1,8 +1,11 @@ #!/usr/bin/env python """ - +Primary Author: Joshua Dawes - CERN, CMS - The University of Manchester +Debugging, Integration and Maintenance: +Andres Cardenas - CERN, CMS - Universidad San Francisco + Upload script wrapper - controls the automatic update system. Note: the name of the file follows a different convention to the others because it should be the same as the current upload script name. From 4b799dfd49f55d0fee73f7ec9498f79cad822088 Mon Sep 17 00:00:00 2001 From: Andres Date: Wed, 21 Apr 2021 11:23:48 +0200 Subject: [PATCH 5/5] Improve selection for destination databse and add automatic server selection based on destination database --- .../Utilities/scripts/uploadConditions_v2.py | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/CondCore/Utilities/scripts/uploadConditions_v2.py b/CondCore/Utilities/scripts/uploadConditions_v2.py index 03e84d6866210..718be8c787cad 100755 --- a/CondCore/Utilities/scripts/uploadConditions_v2.py +++ b/CondCore/Utilities/scripts/uploadConditions_v2.py @@ -150,19 +150,32 @@ def runWizard(basename, dataFilename, metadataFilename): inputTag = getInputChoose(inputTags, '0', '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. 0 (you select the first in the list)\ninputTag [0]: ') + databases = { + 'oraprod': 'oracle://cms_orcon_prod/CMS_CONDITIONS', + 'prod': 'oracle://cms_orcon_prod/CMS_CONDITIONS', + 'oradev': 'oracle://cms_orcoff_prep/CMS_CONDITIONS', + 'prep': 'oracle://cms_orcoff_prep/CMS_CONDITIONS', + } + destinationDatabase = '' ntry = 0 - while ( destinationDatabase != 'oracle://cms_orcon_prod/CMS_CONDITIONS' and destinationDatabase != 'oracle://cms_orcoff_prep/CMS_CONDITIONS' ): + print('\nWhich is the destination database where the tags should be exported?') + print('\n%s) %s' % ('oraprod', databases['oraprod'])) + print('\n%s) %s' % ('oradev', databases['oradev'])) + + while ( destinationDatabase not in databases.values() ): if ntry==0: inputMessage = \ - '\nWhich is the destination database where the tags should be exported? \nPossible choices: oracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \ndestinationDatabase: ' + '\nPossible choices: oraprod or oradev \ndestinationDatabase: ' elif ntry==1: inputMessage = \ - '\nPlease choose one of the two valid destinations: \noracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \ -\ndestinationDatabase: ' + '\nPlease choose one of the two valid destinations: oraprod or oradev \ndestinationDatabase: ' else: raise Exception('No valid destination chosen. Bailing out...') - destinationDatabase = getInputRepeat(inputMessage) + + databaseInput = getInputRepeat(inputMessage).lower() + if databaseInput in databases.keys(): + destinationDatabase = databases[databaseInput] ntry += 1 while True: @@ -262,8 +275,7 @@ def parse_arguments(): server_alias_to_url = { "prep" : "https://cms-conddb-dev.cern.ch/cmsDbCondUpload/", "dev" : "https://cms-conddb-dev.cern.ch/cmsDbCondUpload/", - "prod" : "https://cms-conddb.cern.ch/cmsDbCondUpload/", - None : "https://cms-conddb.cern.ch/cmsDbCondUpload/" + "prod" : "https://cms-conddb.cern.ch/cmsDbCondUpload/" } # if prep, prod or None were given, convert to URLs in dictionary server_alias_to_url @@ -281,6 +293,7 @@ def parse_arguments(): if command_line_data.destinationDatabase in database_alias_to_connection.keys(): command_line_data.destinationDatabase = database_alias_to_connection[command_line_data.destinationDatabase] + # use netrc to get username and password try: netrc_file = command_line_data.netrc @@ -378,8 +391,6 @@ def parse_arguments(): metadata_dictionary["userText"] = metadata_dictionary.get("userText")\ if metadata_dictionary.get("userText") != None\ else str(raw_input("Tag's description [can be empty]:")) - # set the server to use to be the default one - metadata_dictionary["server"] = server_alias_to_url[None] # go through command line options and, if they are set, overwrite entries for (option_name, option_value) in command_line_data.__dict__.items(): @@ -413,6 +424,12 @@ def parse_arguments(): if raw_input("\nDo you want to continue? [y/n] ") != "y": exit() + if metadata_dictionary["server"] == None: + if metadata_dictionary["destinationDatabase"] == "oracle://cms_orcoff_prep/CMS_CONDITIONS": + metadata_dictionary["server"] = server_alias_to_url["prep"] + else: + metadata_dictionary["server"] = server_alias_to_url["prod"] + return metadata_dictionary def get_version(url):