From ee70089ce4b0aa5db2298b4396af605ee62e04d4 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Thu, 25 Jan 2024 07:21:13 -0700 Subject: [PATCH 1/4] fix an issue with queue-specific directives --- scripts/lib/CIME/XML/env_batch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py index 80e75db041a..687973415f5 100644 --- a/scripts/lib/CIME/XML/env_batch.py +++ b/scripts/lib/CIME/XML/env_batch.py @@ -328,7 +328,8 @@ def get_batch_directives(self, case, job, overrides=None, output_format='default directive_prefix = None roots = self.get_children("batch_system") - queue = self.get_value("JOB_QUEUE", subgroup=job) + # This must go through the case object + queue = case.get_value("JOB_QUEUE", subgroup=job) if self._batchtype != "none" and not queue in self._get_all_queue_names(): unknown_queue = True qnode = self.get_default_queue() From 8cc35715312c476a3785561f544338a455b49335 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Fri, 26 Jan 2024 09:43:08 -0700 Subject: [PATCH 2/4] all srt now passing on derecho --- config/cesm/machines/config_machines.xml | 2 +- scripts/fortran_unit_testing/run_tests.py | 2 +- scripts/lib/CIME/SystemTests/system_tests_common.py | 1 + scripts/lib/CIME/XML/machines.py | 6 +++--- scripts/lib/CIME/code_checker.py | 12 +++++++----- scripts/lib/CIME/namelist.py | 1 + 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml index 287e5152267..0bf72bf1c2f 100644 --- a/config/cesm/machines/config_machines.xml +++ b/config/cesm/machines/config_machines.xml @@ -996,7 +996,7 @@ This allows using a different mpirun command to launch unit tests NCAR AMD EPYC de.*.hpc.ucar.edu CNL - intel + intel,gnu mpich $ENV{SCRATCH} diff --git a/scripts/fortran_unit_testing/run_tests.py b/scripts/fortran_unit_testing/run_tests.py index dab5da396fe..29b14f1043d 100755 --- a/scripts/fortran_unit_testing/run_tests.py +++ b/scripts/fortran_unit_testing/run_tests.py @@ -130,7 +130,7 @@ def parse_command_line(args): args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) output = Printer(color=args.color) - + # pylint: disable=broad-exception-raised if args.xml_test_list is None and args.test_spec_dir is None: output.print_error( "You must specify either --test-spec-dir or --xml-test-list." diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py index 0a38c49ad02..67a6ec629cd 100644 --- a/scripts/lib/CIME/SystemTests/system_tests_common.py +++ b/scripts/lib/CIME/SystemTests/system_tests_common.py @@ -399,6 +399,7 @@ def _compare_baseline(self): """ compare the current test output to a baseline result """ + memlist = None with self._test_status: # compare baseline success, comments = compare_baseline(self._case) diff --git a/scripts/lib/CIME/XML/machines.py b/scripts/lib/CIME/XML/machines.py index dc42579515e..5c2f9c33ff3 100644 --- a/scripts/lib/CIME/XML/machines.py +++ b/scripts/lib/CIME/XML/machines.py @@ -252,7 +252,7 @@ def is_valid_compiler(self,compiler): """ Check the compiler is valid for the current machine - >>> machobj = Machines(machine="cori-haswell") + >>> machobj = Machines(machine="derecho") >>> machobj.get_default_compiler() 'intel' >>> machobj.is_valid_compiler("gnu") @@ -266,7 +266,7 @@ def is_valid_MPIlib(self, mpilib, attributes=None): """ Check the MPILIB is valid for the current machine - >>> machobj = Machines(machine="cori-haswell") + >>> machobj = Machines(machine="derecho") >>> machobj.is_valid_MPIlib("mpi-serial") True >>> machobj.is_valid_MPIlib("fake-mpi") @@ -279,7 +279,7 @@ def has_batch_system(self): """ Return if this machine has a batch system - >>> machobj = Machines(machine="cori-haswell") + >>> machobj = Machines(machine="derecho") >>> machobj.has_batch_system() True >>> machobj.set_machine("melvin") diff --git a/scripts/lib/CIME/code_checker.py b/scripts/lib/CIME/code_checker.py index e25b6ce163e..96b780ab2f6 100644 --- a/scripts/lib/CIME/code_checker.py +++ b/scripts/lib/CIME/code_checker.py @@ -15,16 +15,18 @@ ############################################################################### def _run_pylint(on_file, interactive): ############################################################################### - pylint = find_executable("pylint") + pylintstr = find_executable("pylint") cmd_options = " --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import" cmd_options += ",fixme,broad-except,bare-except,eval-used,exec-used,global-statement" cmd_options += ",logging-format-interpolation,no-name-in-module,unspecified-encoding" cmd_options += ",arguments-renamed,no-member,redefined-outer-name" - cimeroot = get_cime_root() + cmd_options += ",deprecated-module,deprecated-class" - if "scripts/Tools" in on_file: - cmd_options +=",relative-import" + cimeroot = get_cime_root() + +# if "scripts/Tools" in on_file: +# cmd_options +=",relative-import" # add init-hook option cmd_options += " --init-hook='sys.path.extend((\"%s\",\"%s\",\"%s\"))'"%\ @@ -32,7 +34,7 @@ def _run_pylint(on_file, interactive): os.path.join(cimeroot,"scripts","Tools"), os.path.join(cimeroot,"scripts","fortran_unit_testing","python")) - cmd = "%s %s %s" % (pylint, cmd_options, on_file) + cmd = "%s %s %s" % (pylintstr, cmd_options, on_file) logger.debug("pylint command is %s"%cmd) stat, out, err = run_cmd(cmd, verbose=False, from_dir=cimeroot) if stat != 0: diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index f8874810e54..7dc192a47e6 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -863,6 +863,7 @@ def shouldRaise(eclass, method, *args, **kw): if not isinstance(e, eclass): raise return + # pylint: disable=broad-exception-raised raise Exception("Expected exception %s not raised" % str(eclass)) From 1d9975c04bf42f345bb4911b5dbe594d8e2b67f3 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Fri, 26 Jan 2024 15:34:56 -0700 Subject: [PATCH 3/4] back out changes in code_checker.py --- scripts/lib/CIME/code_checker.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/lib/CIME/code_checker.py b/scripts/lib/CIME/code_checker.py index 96b780ab2f6..e25b6ce163e 100644 --- a/scripts/lib/CIME/code_checker.py +++ b/scripts/lib/CIME/code_checker.py @@ -15,18 +15,16 @@ ############################################################################### def _run_pylint(on_file, interactive): ############################################################################### - pylintstr = find_executable("pylint") + pylint = find_executable("pylint") cmd_options = " --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import" cmd_options += ",fixme,broad-except,bare-except,eval-used,exec-used,global-statement" cmd_options += ",logging-format-interpolation,no-name-in-module,unspecified-encoding" cmd_options += ",arguments-renamed,no-member,redefined-outer-name" - cmd_options += ",deprecated-module,deprecated-class" - cimeroot = get_cime_root() - -# if "scripts/Tools" in on_file: -# cmd_options +=",relative-import" + + if "scripts/Tools" in on_file: + cmd_options +=",relative-import" # add init-hook option cmd_options += " --init-hook='sys.path.extend((\"%s\",\"%s\",\"%s\"))'"%\ @@ -34,7 +32,7 @@ def _run_pylint(on_file, interactive): os.path.join(cimeroot,"scripts","Tools"), os.path.join(cimeroot,"scripts","fortran_unit_testing","python")) - cmd = "%s %s %s" % (pylintstr, cmd_options, on_file) + cmd = "%s %s %s" % (pylint, cmd_options, on_file) logger.debug("pylint command is %s"%cmd) stat, out, err = run_cmd(cmd, verbose=False, from_dir=cimeroot) if stat != 0: From 15ddca25b79a5971331a958fb03c7a91b6c094eb Mon Sep 17 00:00:00 2001 From: James Edwards Date: Mon, 29 Jan 2024 07:13:32 -0700 Subject: [PATCH 4/4] revert pylint changes --- scripts/fortran_unit_testing/run_tests.py | 2 +- scripts/lib/CIME/code_checker.py | 2 +- scripts/lib/CIME/namelist.py | 1 - scripts/tests/scripts_regression_tests.py | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/fortran_unit_testing/run_tests.py b/scripts/fortran_unit_testing/run_tests.py index 29b14f1043d..63efa914e0d 100755 --- a/scripts/fortran_unit_testing/run_tests.py +++ b/scripts/fortran_unit_testing/run_tests.py @@ -130,7 +130,7 @@ def parse_command_line(args): args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) output = Printer(color=args.color) - # pylint: disable=broad-exception-raised + if args.xml_test_list is None and args.test_spec_dir is None: output.print_error( "You must specify either --test-spec-dir or --xml-test-list." diff --git a/scripts/lib/CIME/code_checker.py b/scripts/lib/CIME/code_checker.py index e25b6ce163e..6c3c23570b0 100644 --- a/scripts/lib/CIME/code_checker.py +++ b/scripts/lib/CIME/code_checker.py @@ -15,7 +15,7 @@ ############################################################################### def _run_pylint(on_file, interactive): ############################################################################### - pylint = find_executable("pylint") + pylint = find_executable("pylint") cmd_options = " --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import" cmd_options += ",fixme,broad-except,bare-except,eval-used,exec-used,global-statement" diff --git a/scripts/lib/CIME/namelist.py b/scripts/lib/CIME/namelist.py index 7dc192a47e6..f8874810e54 100644 --- a/scripts/lib/CIME/namelist.py +++ b/scripts/lib/CIME/namelist.py @@ -863,7 +863,6 @@ def shouldRaise(eclass, method, *args, **kw): if not isinstance(e, eclass): raise return - # pylint: disable=broad-exception-raised raise Exception("Expected exception %s not raised" % str(eclass)) diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index b155106e09b..1a921ac7177 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -2764,7 +2764,7 @@ def check_for_pylint(): from distutils.spawn import find_executable pylint = find_executable("pylint") if pylint is not None: - output = run_cmd_no_fail("pylint --version") + output = run_cmd_no_fail(pylint+" --version") pylintver = re.search(r"pylint\s+(\d+)[.](\d+)[.](\d+)", output) major = int(pylintver.group(1)) minor = int(pylintver.group(2))