Add PCMDI Diags to zppy

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 exclude: "docs|node_modules|migrations|.git|.tox"
-default_stages: [commit]
+default_stages: [pre-commit]
 fail_fast: true
 
 repos:

conda/chrisalis.yml

@@ -0,0 +1,34 @@
+name: zppy_dev
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  # Base
+  # =================
+  - python=3.9.13
+  - pip=22.2.2
+  - configobj=5.0.6
+  - jinja2=3.1.2
+  - mache>=1.5.0
+  - mpas_tools>=0.15.0
+  - pillow=9.2.0
+  # Developer Tools
+  # =================
+  # If versions are updated, also update 'rev' in `.pre-commit-config.yaml`
+  - black=22.8.0        # version from https://anaconda.org/conda-forge/black
+  - flake8=5.0.4        # version from https://anaconda.org/conda-forge/flake8
+  # This line also implicitly installs isort
+  - flake8-isort=4.2.0  # version from https://anaconda.org/conda-forge/flake8-isort
+  - mypy=0.982          # version from https://anaconda.org/conda-forge/mypy
+  - pre-commit=2.20.0   # version from https://anaconda.org/conda-forge/pre-commit
+  - tbump=6.9.0
+  # Documentation
+  # If versions are updated, also update in `.github/workflows/build_workflow.yml`
+  # =================
+  - sphinx=5.2.3
+  - sphinx-multiversion=0.2.4
+  - sphinx_rtd_theme=1.0.0
+  # Need to pin docutils because 0.17 has a bug with unordered lists
+  # https://github.com/readthedocs/sphinx_rtd_theme/issues/1115
+  - docutils=0.16
+prefix: /home/ac.szhang/.conda/envs/zppy_dev

setup.py

@@ -18,7 +18,7 @@ def package_files(directory, prefixes, extensions):
 
 
 data_files = package_files(
-    "zppy", prefixes=[], extensions=["bash", "csh", "cfg", "ini", "sh", "json"]
+    "zppy", prefixes=[], extensions=["bash", "csh", "cfg", "ini", "sh", "json", "py"]
 )
 
 setup(

zppy/__main__.py

@@ -16,6 +16,7 @@
 from zppy.ilamb import ilamb
 from zppy.logger import _setup_custom_logger
 from zppy.mpas_analysis import mpas_analysis
+from zppy.pcmdi_diags import pcmdi_diags
 from zppy.tc_analysis import tc_analysis
 from zppy.ts import ts
 from zppy.utils import check_status, submit_script
@@ -243,6 +244,9 @@ def _launch_scripts(config: ConfigObj, script_dir, job_ids_file, plugins) -> Non
     # ilamb tasks
     existing_bundles = ilamb(config, script_dir, existing_bundles, job_ids_file)
 
+    # pcmdi_diags tasks
+    existing_bundles = pcmdi_diags(config, script_dir, existing_bundles, job_ids_file)
+
     # zppy external plugins
     for plugin in plugins:
         # Get plugin module function

zppy/defaults/default.ini

@@ -107,9 +107,11 @@ input_component = string(default="")
 [ts]
 area_nm = string(default="area")
 cmip_metadata = string(default="inclusions/e3sm_to_cmip/default_metadata.json")
+cmip_plevdata = string(default="inclusions/e3sm_to_cmip/vrt_remap_plev19.nc")
 # Days per file
 dpf = integer(default=30)
 extra_vars = string(default="")
+interp_vars = string(default="U,V,T,Q,RELHUM,OMEGA,Z3")
 # Time-steps per day
 tpd = integer(default=1)
 ts_fmt = string(default="ts_only")
@@ -119,18 +121,230 @@ input_component = string(default="")
   [[__many__]]
   area_nm = string(default=None)
   cmip_metadata = string(default=None)
+  cmip_plevdata = string(default=None)
   dpf = integer(default=None)
   extra_vars = string(default=None)
   tpd = integer(default=None)
   ts_fmt = string(default=None)
   input_component = string(default=None)
+  interp_vars = string(default=None)
 
 [tc_analysis]
 # NOTE: always overrides value in [default]
 input_files = string(default="eam.h2")
 # The scratch directory
 scratch = string(default="")
 
+[pcmdi_diags]
+backend = string(default="mpl")
+cfg = string(default="")
+# File of cmip variable lists (cmip6 convention)
+cmip_variables = string(default="pcmdi_diags/cmip_variables.json")
+# File of specified regions for mean climate calculation
+regions_specs = string(default="pcmdi_diags/regions_specs.json")
+# File of derived variables
+derived_variable = string(default="pcmdi_diags/derived_variable.json")
+# File of observation data name for mean climate calculation
+reference_alias = string(default="pcmdi_diags/reference_alias.json")
+# File of fuction to generate land/sea mask
+process_sftlf = string(default="pcmdi_diags/process_sftlf.py")
+# File of fuction to generate mean climate metrics figure
+clim_plot_parser = string(default="pcmdi_diags/mean_climate_plot_parser.py")
+# File of module to plot mean climate metrics figure
+clim_plot_driver = string(default="pcmdi_diags/mean_climate_plot_driver.py")
+# Path to observation time-series data
+# Required for "mean_climate","variability_mode","enso"
+obs_ts = string(default="")
+# observational data sets (see reference_alias.json)
+# observation data tag in reference_alias
+obs_sets = string(default="default")
+# options specific for constructing pcmdi preferred file name conventions
+# required for "model_vs_obs" comparison
+cmip_name = string(default="e3sm.historical.v3-LR.0051")
+# required for "model_vs_model" comparison
+cmip_name_ref = string(default="e3sm.historical.v3-LR.0051")
+# options shared by pcmdi
+pmp_debug = string(default=False)
+# flag to process the land/sea mask within pcmdi
+generate_sftlf = string(default=True)
+# variables to be used by the pcmdi diagnostics
+# needs to setup for each subsections
+vars = string(default="")
+# sets of diagnostics from pcmdi package
+sets = string_list(default=list("mean_climate","variability_mode_atm","variability_mode_cpl","enso"))
+# options to identify subset of pcmdi drivers ("mean_climate","variability_mode","enso")
+subset = string(default="")
+#options for cmip model metrics data from pcmdi
+#group of pcmdi generated cmip metrics data (mip.exp.version)
+pcmdi_data_set=string(default="cmip6.historical.v20220928")
+# path to pcmdi generated cmip metrics data
+pcmdi_data_path=string(default="")
+##########################################################################################
+# below followed the setup in e3sm_diag but used for PCMDI workflow
+##########################################################################################
+# See url<need to work on document later>
+multiprocessing = boolean(default=True)
+# See url<need to work on document later>
+num_workers = integer(default=24)
+# See url<need to work on document later>
+figure_format = string(default="png")
+# comparision type (same as e3sm_diag)
+run_type = string(default="model_vs_obs")
+# Used to label the results directory
+# Options are "model_vs_obs" and "model_vs_model"
+tag = string(default="model_vs_obs")
+###########################################################################################
+# Required for run_type="model_vs_model" runs, different from e3sm_diag,
+# model_vs_model in pcmdi referred to the comparision of two model simulations
+# with observations and cmip models.
+###########################################################################################
+#path for reference model data (time series)
+reference_data_path_ts = string(default="")
+# pcmdi_diags.py will set to match `years` if not specified
+ref_years = string_list(default=list(""))
+# End year (i.e., the last year to use) for the reference data
+ref_end_yr = string(default="")
+# Final year (i.e., the last available year) for the reference data
+ref_final_yr = string(default="")
+# Start year for the reference data
+ref_start_yr = string(default="")
+# reference model name
+ref_name = string(default="")
+# The years increment for reference data
+ts_num_years_ref = integer(default=5)
+# Set to true to swap test and ref when run_type="model_vs_model"
+swap_test_ref = boolean(default=False)
+##########################################################################################
+# options for pcmdi mode varibility diagnostics
+# vars = "psl" for atm_modes
+# vars = "ts" for cpl_modes
+#########################################################################################
+#name of atmospheric modes varibility
+atm_modes = string_list(default=list("NAM","NAO","PNA","NPO","SAM","PSA1","PSA2"))
+#name of coupled modes varibility
+cpl_modes = string_list(default=list("PDO","NPGO","AMO"))
+#keywards for unit conversion in pcmdi (model)
+ModUnitsAdjust = string(default="")
+#keywards for unit conversion in pcmdi (observation)
+ObsUnitsAdjust = string(default="")
+#frequency of the model data
+frequency = string(default="mo")
+#options specific for mode varibility metrics in pcmdi
+seasons = string(default="monthly")
+landmask = string(default=False)
+RmDomainMean = string(default=True)
+EofScaling = string(default=False)
+ConvEOF = string(default=True)
+CBF = string(default=True)
+cmec = string(default=True)
+update_json = string(default=False)
+plot_obs = string(default=True)
+plot = string(default=True)
+nc_out_obs = string(default=True)
+nc_out = string(default=True)
+##########################################################################################
+# options for pcmdi enso diagnostics
+# vars = "psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt"
+##########################################################################################
+groups = string_list(default=list("ENSO_perf","ENSO_proc","ENSO_tel"))
+##########################################################################################
+# optional for mean climate diagnostics
+# vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,
+#         rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,
+#         va-850,zg-500"
+##########################################################################################
+# model data grid after remapping
+grid = string(default="180x360_aave")
+#flag to turn on regional mean climate metrics
+regional = string(default="y")
+#default regions for mean climate metrics data
+regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS,NHEX_ocean,SHEX_ocean,NHEX_land,SHEX_land,ocean_50S50N")
+# save derived climatology data
+save_test_clims = string(default=True)
+# method to determine the way to process mean climate data
+# default used nco instead of pcmdi built-in function
+climatology_process_method = string(default="nco")
+# Regridding by pcmdi (default is to regrid data to 2.5x2.5 grid for diagnostic metrics)
+# Required for mean climate
+# OPTIONS: '2.5x2.5' or an actual cdms2 grid object
+target_grid = string(default="2.5x2.5")
+# OPTIONS: String for description on the selected grid
+target_grid_string = string(default="2p5x2p5")
+# OPTIONS: 'regrid2','esmf'
+regrid_tool = string(default="esmf")
+# OPTIONS: 'linear','conservative', only if tool is esmf
+regrid_method = string(default="regrid2")
+# OPTIONS: 'linear','conservative', only if tool is esmf
+regrid_method_ocn = string(default="conservative")
+# setup for parallel coordinate plots (hide makers for sigle model)
+parcord_show_markers = string(default=False)
+# setup for portrait plots (add vertical line to separate test and reference models)
+portrait_vertical_line = string(default=True)
+
+  [[__many__]]
+  backend = string(default=None)
+  cfg = string(default=None)
+  vars = string(default=None)
+  grid = string(default=None)
+  cmip_metadata = string(default=None)
+  cmip_variables = string(default=None)
+  pcmdi_data_set = string(default=None)
+  pcmdi_data_path = string(default=None)
+  derived_variable = string(default=None)
+  reference_alias = string(default=None)
+  regions_specs = string(default=None)
+  process_sftlf = string(default=None)
+  multiprocessing = boolean(default=None)
+  num_workers = integer(default=None)
+  obs_ts = string(default=None)
+  figure_format = string(default=None)
+  ref_end_yr = string(default=None)
+  ref_final_yr = string(default=None)
+  ref_name = string(default=None)
+  ref_start_yr = string(default=None)
+  ref_years = string_list(default=None)
+  reference_data_path_ts = string(default=None)
+  run_type = string(default=None)
+  sets = string_list(default=None)
+  swap_test_ref = boolean(default=None)
+  tag = string(default=None)
+  ts_num_years_ref = integer(default=None)
+  climatology_process_method = string(default=None)
+  target_grid = string(default=None)
+  target_grid_string = string(default=None)
+  regrid_tool = string(default=None)
+  regrid_method = string(default=None)
+  regrid_method_ocn = string(default=None)
+  obs_sets = string(default=None)
+  regions = string(default=None)
+  regional = string(default=None)
+  save_test_clims = string(default=None)
+  seasons = string(default=None)
+  RmDomainMean = string(default=None)
+  EofScaling = string(default=None)
+  ConvEOF = string(default=None)
+  CBF = string(default=None)
+  cmec = string(default=None)
+  update_json = string(default=None)
+  subset = string(default=None)
+  landmask = string(default=None)
+  frequency = string(default=None)
+  generate_sftlf = string(default=None)
+  atm_modes = string_list(default=None)
+  cpl_modes = string_list(default=None)
+  groups = string_list(default=None)
+  ModUnitsAdjust = string(default=None)
+  ObsUnitsAdjust = string(default=None)
+  cmip_name = string(default=None)
+  cmip_name_ref = string(default=None)
+  pmp_debug = string(default=None)
+  nc_out_obs = string(default=None)
+  nc_out = string(default=None)
+  plot_obs = string(default=None)
+  plot = string(default=None)
+  parcord_show_markers = string(default=None)
+  portrait_vertical_line = string(default=None)
+
 [e3sm_diags]
 # See https://e3sm-project.github.io/e3sm_diags/_build/html/master/available-parameters.html
 backend = string(default="mpl")