From f7fc2684287172879420c3d8f64f6d4f1e610f9a Mon Sep 17 00:00:00 2001 From: Geovanny Gonzalez Date: Thu, 28 Nov 2024 14:53:09 +0100 Subject: [PATCH] Execute `cmsRun` to validate driver parameters Instead of just creating the Python configuration, execute `cmsRun` and simulate a few events to cross-check the driver parameters. Allow users to set this behavior in the `RelVal` attributes. --- core/controller/relval_controller.py | 1 + core/model/relval.py | 21 +++++++++++++ core/model/relval_step.py | 8 +++-- scripts/add_execute_steps.py | 33 +++++++++++++++++++++ vue_frontend/src/components/RelVals.vue | 1 + vue_frontend/src/components/RelValsEdit.vue | 11 +++++++ 6 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 scripts/add_execute_steps.py diff --git a/core/controller/relval_controller.py b/core/controller/relval_controller.py index c695d7b..627a272 100644 --- a/core/controller/relval_controller.py +++ b/core/controller/relval_controller.py @@ -135,6 +135,7 @@ def get_editing_info(self, obj): editing_info["workflow_id"] = False editing_info["workflow_name"] = is_new editing_info["steps"] = is_new + editing_info["execute_steps"] = True return editing_info diff --git a/core/model/relval.py b/core/model/relval.py index b4e41e7..769d1cb 100644 --- a/core/model/relval.py +++ b/core/model/relval.py @@ -59,6 +59,9 @@ class RelVal(ModelBase): 'workflow_name': '', # ReqMgr2 names 'workflows': [], + # Execute `cmsRun` when preparing the + # configuration file + 'execute_steps': False, } lambda_checks = { @@ -323,3 +326,21 @@ def get_campaign(self): return f'{cmssw_release}__{batch_name}-{campaign_timestamp}' return f'{cmssw_release}__{batch_name}' + + def execute_steps_for_configuration(self) -> bool: + """ + Determine if the `cmsRun` command should be executed + when generating the Python configuration. + """ + execute_steps = None + try: + execute_steps = self.get('execute_steps') + except Exception as e: + self.logger.error("Unable to retrieve the execute_steps attribute: %s", e) + + self.logger.debug('Execute steps is set: %s', execute_steps) + # If the document does not set it + if not isinstance(execute_steps, bool): + return False + + return execute_steps diff --git a/core/model/relval_step.py b/core/model/relval_step.py index e0ebc99..2f35cc5 100644 --- a/core/model/relval_step.py +++ b/core/model/relval_step.py @@ -321,8 +321,12 @@ def get_command(self, custom_fragment=None, for_submission=False): if custom_fragment: arguments_dict['fragment_name'] = custom_fragment - # No execution - arguments_dict['no_exec'] = True + # Enable the execution of `cmsRun` for a quick + # validation + execute_cmsrun = self.parent().execute_steps_for_configuration() + if not execute_cmsrun: + arguments_dict['no_exec'] = True + # Handle input/output file names arguments_dict['fileout'] = f'"file:step{index + 1}.root"' arguments_dict['python_filename'] = f'{self.get_config_file_name()}.py' diff --git a/scripts/add_execute_steps.py b/scripts/add_execute_steps.py new file mode 100644 index 0000000..8b3bf1a --- /dev/null +++ b/scripts/add_execute_steps.py @@ -0,0 +1,33 @@ +""" +This script sets the new attribute +`execute_steps` on every `RelVal` object that +does not include it already. +""" +import sys +import os.path +import os +# pylint: disable-next=wrong-import-position +sys.path.append(os.path.abspath(os.path.pardir)) +from core_lib.database.database import Database + +# Configure the database client +mongo_db_username = os.getenv("MONGO_DB_USERNAME", "") +mongo_db_password = os.getenv("MONGO_DB_PASSWORD", "") +mongo_db_host = os.getenv("MONGO_DB_HOST", "") +mongo_db_port = int(os.getenv("MONGO_DB_PORT", "27017")) +Database.set_host_port(host=mongo_db_host, port=mongo_db_port) +Database.set_credentials(username=mongo_db_username, password=mongo_db_password) +Database.set_database_name('relval') + +database = Database('relvals') +total_entries = database.get_count() +print('Total entries: %s' % (total_entries)) + +for index, item in enumerate(database.query(limit=total_entries)): + print('Processing entry %s/%s %s' % (index + 1, total_entries, item.get('prepid', ''))) + execute_steps = item.get('execute_steps') + if execute_steps is None: + item['execute_steps'] = False + database.save(item) + +print('Done') diff --git a/vue_frontend/src/components/RelVals.vue b/vue_frontend/src/components/RelVals.vue index f4ca3cf..ad65052 100644 --- a/vue_frontend/src/components/RelVals.vue +++ b/vue_frontend/src/components/RelVals.vue @@ -237,6 +237,7 @@ export default { {'dbName': 'status', 'displayName': 'Status', 'visible': 1, 'sortable': true}, {'dbName': 'batch_name', 'displayName': 'Batch Name', 'visible': 1, 'sortable': true}, {'dbName': 'cmssw_release', 'displayName': 'CMSSW Release', 'visible': 1, 'sortable': true}, + {'dbName': 'execute_steps', 'displayName': 'Execute cmsRun', 'visible': 1}, {'dbName': 'cpu_cores', 'displayName': 'CPU Cores', 'visible': 1, 'sortable': true}, {'dbName': 'matrix', 'displayName': 'Matrix', 'visible': 1, 'sortable': true}, {'dbName': 'memory', 'displayName': 'Memory', 'visible': 1, 'sortable': true}, diff --git a/vue_frontend/src/components/RelValsEdit.vue b/vue_frontend/src/components/RelValsEdit.vue index 4a849aa..7253a17 100644 --- a/vue_frontend/src/components/RelValsEdit.vue +++ b/vue_frontend/src/components/RelValsEdit.vue @@ -262,6 +262,17 @@ Add step {{listLength(editableObject.steps) + 1}} + + Execute cmsRun for steps + + + + Time per event s