add updates for #573, #577, #566

cime_config/SystemTests/sct.py

@@ -66,12 +66,12 @@ def _component_compare_test(self, suffix1, suffix2,
                                 ignore_fieldlist_diffs=False):
         with self._test_status:
             stat,netcdf_filename,err=run_cmd('ls ./run/case2run/*h1*8400.nc ')
-            stat,DIFFs,err=run_cmd('ncdump -ff -p 9,17 -v QDIFF,TDIFF '+netcdf_filename+' | egrep //\.\*DIFF | sed s/^\ \*// | sed s/\[,\;\].\*// | sed s/^0/0.0/  | uniq')
+            stat,DIFFs,err=run_cmd('ncdump -ff -p 9,17 -v QDIFF,TDIFF '+netcdf_filename+' | egrep //\.\*DIFF | sed s/^\ \*// | sed s/^0,/0.0,/  | sed s/^0\;/0.0\;/ | sed s/\[,\;\].\*// |  uniq')
             array_of_DIFFs=DIFFs.split("\n")
             answer=max([abs(float(x)) for x in array_of_DIFFs])
             comments="Checking QDIFF,TDIFF in SCAM run."
             append_testlog(comments, self._orig_caseroot)
-            # Test for greater that round off changes.
+            # Test for greater than round off changes.
             if answer < 1e-10:
                 self._test_status.set_status("{}_{}_{}".format(COMPARE_PHASE, self._run_one_suffix, self._run_two_suffix), TEST_PASS_STATUS)
                 comments="QDIFF,TDIFF: PASS"

cime_config/testdefs/testlist_cam.xml

@@ -1401,16 +1401,49 @@
       <machine name="cheyenne" compiler="intel" category="camchem"/>
     </machines>
   </test>
-  <test compset="F2000climo" grid="f09_f09_mg17" name="ERP_Ln9_Vnuopc" testmods="cam/outfrq9s_mg3_casper">
+  <test compset="F2000dev" grid="f09_f09_mg17" name="ERP_Ln9_Vnuopc" testmods="cam/outfrq9s_mg3_default">
     <machines>
       <machine name="casper" compiler="pgi-gpu"   category="casper_gpu"/>
       <machine name="casper" compiler="nvhpc-gpu" category="casper_gpu"/>
-      <machine name="casper" compiler="intel"     category="casper_cpu"/>
-      <machine name="casper" compiler="pgi"       category="casper_cpu"/>
-      <machine name="casper" compiler="nvhpc"     category="casper_cpu"/>
     </machines>
     <options>
-      <option name="wallclock">00:30:00</option>
+      <option name="wallclock">00:59:00</option>
+    </options>
+  </test>
+  <test compset="F2000dev" grid="f09_f09_mg17" name="ERP_Ln9_Vnuopc" testmods="cam/outfrq9s_mg2_default">
+    <machines>
+      <machine name="casper" compiler="pgi-gpu"   category="casper_gpu"/>
+      <machine name="casper" compiler="nvhpc-gpu" category="casper_gpu"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:59:00</option>
+    </options>
+  </test>
+  <test compset="F2000dev" grid="f09_f09_mg17" name="ERP_Ln9_Vnuopc" testmods="cam/outfrq9s_mg3_nondefault">
+    <machines>
+      <machine name="casper" compiler="pgi-gpu"   category="casper_gpu"/>
+      <machine name="casper" compiler="nvhpc-gpu" category="casper_gpu"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:59:00</option>
+    </options>
+  </test>
+  <test compset="F2000dev" grid="f09_f09_mg17" name="ERP_Ln9_Vnuopc" testmods="cam/outfrq9s_mg3_pcols1536">
+    <machines>
+      <machine name="casper" compiler="pgi-gpu"   category="casper_gpu"/>
+      <machine name="casper" compiler="nvhpc-gpu" category="casper_gpu"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:59:00</option>
+    </options>
+  </test>
+  <test compset="F2000dev" grid="f09_f09_mg17" name="ERP_Ln9_Vnuopc_G4" testmods="cam/outfrq9s_mg3_default">
+    <machines>
+      <machine name="casper" compiler="pgi-gpu"   category="casper_gpu"/>
+      <machine name="casper" compiler="nvhpc-gpu" category="casper_gpu"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:59:00</option>
     </options>
   </test>
   <test compset="QPC4" grid="f19_f19_mg17" name="ERP_Ln9_Vnuopc" testmods="cam/outfrq9s">

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/shell_commands

@@ -0,0 +1,7 @@
+./xmlchange NTASKS=36
+./xmlchange NTHRDS=1
+./xmlchange ROOTPE='0'
+./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange TIMER_DETAIL='6'
+./xmlchange TIMER_LEVEL='999'

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_casper/user_nl_clm → cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/user_nl_clm

@@ -24,4 +24,3 @@
 hist_nhtfrq = 9
 hist_mfilt  = 1
 hist_ndens  = 1
-

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3/shell_commands

@@ -2,5 +2,5 @@
 ./xmlchange GLC_NCPL=\$ATM_NCPL
 ./xmlchange CAM_CONFIG_OPTS=' -microphys mg3' --append
 if [ "`./xmlquery ATM_GRID --value`" == "C96" ]; then
-  ./xmlchange NTASKS=-1
+  ./xmlchange NTASKS=-2
 fi

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_default/shell_commands

@@ -0,0 +1,8 @@
+./xmlchange NTASKS=36
+./xmlchange NTHRDS=1
+./xmlchange ROOTPE='0'
+./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange CAM_CONFIG_OPTS=' -microphys mg3' --append
+./xmlchange TIMER_DETAIL='6'
+./xmlchange TIMER_LEVEL='999'

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_default/user_nl_cam

@@ -0,0 +1,4 @@
+mfilt=1,1,1,1,1,1
+ndens=1,1,1,1,1,1
+nhtfrq=9,9,9,9,9,9
+inithist='ENDOFRUN'

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_default/user_nl_clm

@@ -0,0 +1,26 @@
+!----------------------------------------------------------------------------------
+! Users should add all user specific namelist changes below in the form of 
+! namelist_var = new_namelist_value 
+!
+! Include namelist variables for drv_flds_in ONLY if -megan and/or -drydep options
+! are set in the CLM_NAMELIST_OPTS env variable.
+!
+! EXCEPTIONS: 
+! Set use_cndv           by the compset you use and the CLM_BLDNML_OPTS -dynamic_vegetation setting
+! Set use_vichydro       by the compset you use and the CLM_BLDNML_OPTS -vichydro           setting
+! Set use_cn             by the compset you use and CLM_BLDNML_OPTS -bgc  setting
+! Set use_crop           by the compset you use and CLM_BLDNML_OPTS -crop setting
+! Set spinup_state       by the CLM_BLDNML_OPTS -bgc_spinup      setting
+! Set irrigate           by the CLM_BLDNML_OPTS -irrig           setting
+! Set dtime              with L_NCPL                             option
+! Set fatmlndfrc         with LND_DOMAIN_PATH/LND_DOMAIN_FILE    options
+! Set finidat            with RUN_REFCASE/RUN_REFDATE/RUN_REFTOD options for hybrid or branch cases
+!                        (includes $inst_string for multi-ensemble cases)
+! Set glc_grid           with CISM_GRID                          option
+! Set glc_smb            with GLC_SMB                            option
+! Set maxpatch_glcmec    with GLC_NEC                            option
+! Set glc_do_dynglacier  with GLC_TWO_WAY_COUPLING               env variable
+!----------------------------------------------------------------------------------
+hist_nhtfrq = 9
+hist_mfilt  = 1
+hist_ndens  = 1

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/shell_commands

@@ -0,0 +1,8 @@
+./xmlchange NTASKS=36
+./xmlchange NTHRDS=1
+./xmlchange ROOTPE='0'
+./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange CAM_CONFIG_OPTS=' -microphys mg3' --append
+./xmlchange TIMER_DETAIL='6'
+./xmlchange TIMER_LEVEL='999'

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/user_nl_cam

@@ -0,0 +1,24 @@
+mfilt=1,1,1,1,1,1
+ndens=1,1,1,1,1,1
+nhtfrq=9,9,9,9,9,9
+inithist='ENDOFRUN'
+micro_mg_do_graupel=.false.
+micro_mg_do_hail=.true.
+micro_do_sb_physics=.true.
+micro_do_massless_droplet_destroyer=.true.
+microp_uniform=.true.
+micro_mg_nccons=.true.
+micro_mg_nicons=.true.
+micro_mg_ngcons=.true.
+micro_mg_nrcons=.true.
+micro_mg_nscons=.true.
+micro_mg_evap_sed_off=.true.
+micro_mg_icenuc_rh_off=.true.
+micro_mg_icenuc_use_meyers=.true.
+micro_mg_evap_scl_ifs=.true.
+micro_mg_evap_rhthrsh_ifs=.true.
+micro_mg_rainfreeze_ifs=.true.
+micro_mg_ifs_sed=.true.
+micro_mg_precip_fall_corr=.true.
+micro_mg_implicit_fall=.false.
+micro_mg_accre_sees_auto=.true.

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/user_nl_clm

@@ -0,0 +1,26 @@
+!----------------------------------------------------------------------------------
+! Users should add all user specific namelist changes below in the form of 
+! namelist_var = new_namelist_value 
+!
+! Include namelist variables for drv_flds_in ONLY if -megan and/or -drydep options
+! are set in the CLM_NAMELIST_OPTS env variable.
+!
+! EXCEPTIONS: 
+! Set use_cndv           by the compset you use and the CLM_BLDNML_OPTS -dynamic_vegetation setting
+! Set use_vichydro       by the compset you use and the CLM_BLDNML_OPTS -vichydro           setting
+! Set use_cn             by the compset you use and CLM_BLDNML_OPTS -bgc  setting
+! Set use_crop           by the compset you use and CLM_BLDNML_OPTS -crop setting
+! Set spinup_state       by the CLM_BLDNML_OPTS -bgc_spinup      setting
+! Set irrigate           by the CLM_BLDNML_OPTS -irrig           setting
+! Set dtime              with L_NCPL                             option
+! Set fatmlndfrc         with LND_DOMAIN_PATH/LND_DOMAIN_FILE    options
+! Set finidat            with RUN_REFCASE/RUN_REFDATE/RUN_REFTOD options for hybrid or branch cases
+!                        (includes $inst_string for multi-ensemble cases)
+! Set glc_grid           with CISM_GRID                          option
+! Set glc_smb            with GLC_SMB                            option
+! Set maxpatch_glcmec    with GLC_NEC                            option
+! Set glc_do_dynglacier  with GLC_TWO_WAY_COUPLING               env variable
+!----------------------------------------------------------------------------------
+hist_nhtfrq = 9
+hist_mfilt  = 1
+hist_ndens  = 1

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/shell_commands

@@ -0,0 +1,8 @@
+./xmlchange NTASKS=36
+./xmlchange NTHRDS=1
+./xmlchange ROOTPE='0'
+./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange CAM_CONFIG_OPTS=' -microphys mg3 -pcols 1536' --append
+./xmlchange TIMER_DETAIL='6'
+./xmlchange TIMER_LEVEL='999'

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/user_nl_cam

@@ -0,0 +1,4 @@
+mfilt=1,1,1,1,1,1
+ndens=1,1,1,1,1,1
+nhtfrq=9,9,9,9,9,9
+inithist='ENDOFRUN'

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/user_nl_clm

@@ -0,0 +1,26 @@
+!----------------------------------------------------------------------------------
+! Users should add all user specific namelist changes below in the form of 
+! namelist_var = new_namelist_value 
+!
+! Include namelist variables for drv_flds_in ONLY if -megan and/or -drydep options
+! are set in the CLM_NAMELIST_OPTS env variable.
+!
+! EXCEPTIONS: 
+! Set use_cndv           by the compset you use and the CLM_BLDNML_OPTS -dynamic_vegetation setting
+! Set use_vichydro       by the compset you use and the CLM_BLDNML_OPTS -vichydro           setting
+! Set use_cn             by the compset you use and CLM_BLDNML_OPTS -bgc  setting
+! Set use_crop           by the compset you use and CLM_BLDNML_OPTS -crop setting
+! Set spinup_state       by the CLM_BLDNML_OPTS -bgc_spinup      setting
+! Set irrigate           by the CLM_BLDNML_OPTS -irrig           setting
+! Set dtime              with L_NCPL                             option
+! Set fatmlndfrc         with LND_DOMAIN_PATH/LND_DOMAIN_FILE    options
+! Set finidat            with RUN_REFCASE/RUN_REFDATE/RUN_REFTOD options for hybrid or branch cases
+!                        (includes $inst_string for multi-ensemble cases)
+! Set glc_grid           with CISM_GRID                          option
+! Set glc_smb            with GLC_SMB                            option
+! Set maxpatch_glcmec    with GLC_NEC                            option
+! Set glc_do_dynglacier  with GLC_TWO_WAY_COUPLING               env variable
+!----------------------------------------------------------------------------------
+hist_nhtfrq = 9
+hist_mfilt  = 1
+hist_ndens  = 1

doc/ChangeLog

@@ -10,6 +10,9 @@ Purpose of changes (include the issue number and title text for each relevant Gi
 - Update to cesm2_3_alpha09a externals (https://github.com/ESCOMP/CAM/issues/565)
 - CIME update needed for running the pumas codes on GPU and introduce a GPU
   regression test suite (https://github.com/ESCOMP/CAM/issues/512)
+- F2000climo using C96 grid crashes intermittently (https://github.com/ESCOMP/CAM/issues/573)
+- Failing SCT test (https://github.com/ESCOMP/CAM/issues/566)
+- PUMAS GPU regression test suite (https://github.com/ESCOMP/CAM/issues/577)
 
 Describe any changes made to build system: N/A
 
@@ -21,9 +24,26 @@ Describe any substantial timing or memory changes: N/A
 
 Code reviewed by: goldy, cacraig, nusbaume
 
-List all files eliminated: N/A
+List all files eliminated:
+D	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_casper/shell_commands
+D	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_casper/user_nl_cam
+D	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_casper/user_nl_clm
+	- remove testmods for unused test
 
-List all files added and what they do: N/A
+List all files added and what they do:
+A	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/shell_commands
+A	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/user_nl_cam
+A	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg2_default/user_nl_clm
+A	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_default/shell_commands
+A	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_default/user_nl_cam
+A	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_default/user_nl_clm
+A	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/shell_commands
+A	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/user_nl_cam
+A	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_nondefault/user_nl_clm
+A	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/shell_commands
+A	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/user_nl_cam
+A	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/user_nl_clm
+	- add testmods for new tests
 
 List all existing files that have been modified, and describe the changes:
 M       Externals.cfg
@@ -38,6 +58,13 @@ M	cime_config/testdefs/testlist_cam.xml
 	  - FSCAM -> 2000_CAM60%SCAM_CLM50%SP_CICE5%PRES_DOCN%DOM_SROF_SGLC_SWAV
 	  - FW1850 -> 1850_CAM60%WCTS_CLM50%SP_CICE5%PRES_DOCN%DOM_MOSART_SGLC_SWAV
 	  - FWHIST -> HIST_CAM60%WCTS_CLM50%SP_CICE5%PRES_DOCN%DOM_MOSART_SGLC_SWAV
+	- add GPU tests for PUMAS on casper
+M	cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3/shell_commands
+	- increase NTASKS for C96 grid that is crashing intermittently
+M	cime_config/SystemTests/sct.py
+	- fix failing SCT test
+M	test/system/test_driver.sh
+	- add casper-specific testing
 If there were any failures reported from running test_driver.sh on any test
 platform, and checkin with these failures has been OK'd by the gatekeeper,
 then copy the lines from the td.*.status files for the failed tests to the