From 63bde3f63e2b922d379db4c70c2bc32822396b12 Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Mon, 21 Oct 2024 13:46:49 +1100 Subject: [PATCH 1/4] solved comflict --- src/mopper/mop_utils.py | 10 ++++++---- tests/test_mop_utils.py | 22 ++++++++++++++++++++-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/mopper/mop_utils.py b/src/mopper/mop_utils.py index b322f18..95745be 100755 --- a/src/mopper/mop_utils.py +++ b/src/mopper/mop_utils.py @@ -1110,9 +1110,10 @@ def define_attrs(ctx): ctx : click context Includes obj dict with 'cmor' settings, exp attributes """ - #var_log = logging.getLogger(ctx.obj['var_log']) - attrs = ctx.obj['attrs'] + var_log = logging.getLogger(ctx.obj['var_log']) + attrs = ctx.obj['attrs'].copy() notes = attrs.get('notes', '') + var_log.debug(f"in define_attrs, notes: {notes}") # open file containing notes fname = import_files('mopdata').joinpath('notes.yaml') data = read_yaml(fname)['notes'] @@ -1125,7 +1126,8 @@ def define_attrs(ctx): fval = ctx.obj[field] for k,v in data[field].items(): if k == fval or (k[0] == '~' and k[1:] in fval): - notes += v + notes += f" {v} " if notes != '': - attrs['notes'] = notes + attrs['notes'] = notes.strip() + var_log.debug(f"in define_attrs, attrs: {attrs}") return attrs diff --git a/tests/test_mop_utils.py b/tests/test_mop_utils.py index dd64697..908fcee 100644 --- a/tests/test_mop_utils.py +++ b/tests/test_mop_utils.py @@ -22,7 +22,8 @@ import logging from pathlib import Path -from mopper.mop_utils import (check_timestamp, get_cmorname,) +from mopper.mop_utils import (check_timestamp, get_cmorname, + define_attrs) ctx = click.Context(click.Command('cmd'), @@ -121,8 +122,25 @@ def test_get_cmorname(caplog): assert iname == 'longitude' assert jname == 'latitude' assert zname == 'plev3' - # test generic axis alevel ctx.obj['axes'] = 'longitude latitude alevel time' with ctx: zname = get_cmorname('theta_model_level_number') assert zname == 'hybrid_height' + +def test_define_attrs(caplog): + global ctx + caplog.set_level(logging.DEBUG, logger='varlog_1') + ctx.obj['attrs'] = {'notes': "some existing note"} + ctx.obj['variable_id'] = "ta" + ctx.obj['calculation'] = "... plevinterp(var[0]) " + with ctx: + out = define_attrs() + assert out['notes'] == "some existing note Linearly interpolated from model levels using numpy.interp() function. NaNs are assigned to pressure levels falling out of the height range covered by the model" + # repeating to make sure we are not using reference to ctx see issue #190 + with ctx: + out = define_attrs() + assert out['notes'] == "some existing note Linearly interpolated from model levels using numpy.interp() function. NaNs are assigned to pressure levels falling out of the height range covered by the model" + ctx.obj['attrs'] = {} + with ctx: + out = define_attrs() + assert out['notes'] == "Linearly interpolated from model levels using numpy.interp() function. NaNs are assigned to pressure levels falling out of the height range covered by the model" From aa470e822604d2434ad7174886385641e1f6a94a Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Wed, 30 Oct 2024 15:52:13 +1100 Subject: [PATCH 2/4] fixed #191 --- ACDD_conf.yaml | 9 ++++++--- CMIP6_conf.yaml | 15 ++++++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/ACDD_conf.yaml b/ACDD_conf.yaml index 1112c81..6af9b7d 100755 --- a/ACDD_conf.yaml +++ b/ACDD_conf.yaml @@ -71,14 +71,17 @@ cmor: # use this to indicate the path used for new or modified tables # these will be used in preference to the package tables tables_path: "" - # ancillary files path - # when running model with payu ancil files are copied to work//INPUT - # you can leave these empty if processing only atmos + # Ancil files are needed only for specific variables when + # grid information is not fully available from output + # you can usually leave these empty if processing only atmos + # ancillary files relative path to or fullpath ancils_path: "localdata/ancils" grid_ocean: "" grid_ice: "" mask_ocean: "" + # to supply land area fraction if not available in output land_frac: "" + # to supply tile area fraction if not available in output tile_frac: "" # defines Controlled Vocabularies and required attributes # leave ACDD to follow NCI publishing requirements diff --git a/CMIP6_conf.yaml b/CMIP6_conf.yaml index 3396e92..e68c8f0 100755 --- a/CMIP6_conf.yaml +++ b/CMIP6_conf.yaml @@ -1,6 +1,8 @@ ################################################################ # USER OPTIONS # Settings to manage cmorisation and set tables/variables to process +# This config file is to produce files to submit to CMIP6 project +# It's more restrictive than ACDD_conf.yaml cmor: # If test true it will just run the setup but not launch the job automatically test: false @@ -62,14 +64,17 @@ cmor: # as you should use official CMOR cmip6 tables for CMIP6 mode # use this only if current package version is obsolete tables_path: "" - # ancillary files path - # when running model with payu ancil files are copied to work//INPUT - # you can leave these empty if processing only atmos - ancils_path: "data/ancils" + # Ancil files are needed only for specific variables when + # grid information is not fully available from output + # you can usually leave these empty if processing only atmos + # ancillary files relative path to or fullpath + ancils_path: "localdata/ancils" grid_ocean: "" - mask_ocean: "" grid_ice: "" + mask_ocean: "" + # to supply land area fraction if not available in output land_frac: "" + # to supply tile area fraction if not available in output tile_frac: "" history_data: '' # DO NOT REMOVE OR ALTER this if you don't know what you're doing :-) From 65c4a8afea37ad1f96050758201d4cf6fede6b42 Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Wed, 30 Oct 2024 16:17:15 +1100 Subject: [PATCH 3/4] Update meta.yaml --- conda/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/meta.yaml b/conda/meta.yaml index a6e78e7..af35098 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,14 +1,14 @@ package: name: mopper #version: "{{ environ.get('GIT_DESCRIBE_TAG', '1.1.0') }}" - version: '1.1.0' + version: '1.1.1' #source: # path: ../. source: #url: https://github.com/ACCESS-Hive/ACCESS-MOPPeR/archive/refs/tags/{{ environ.get('RELEASE_VERSION') }}.tar.gz git_url: https://github.com/ACCESS-Hive/ACCESS-MOPPeR.git - git_tag: "1.1.0" + git_tag: "1.1.1" #git_rev: "1.1.0" #git_depth: 1 # (Defaults to -1/not shallow) From 9f473bd33a8b0dace50814e8e96848c142f2dc61 Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Wed, 30 Oct 2024 16:12:50 +1100 Subject: [PATCH 4/4] merged with remote --- ACDD_conf.yaml | 2 +- CMIP6_conf.yaml | 2 +- README.md | 4 +--- conda/meta.yaml | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ACDD_conf.yaml b/ACDD_conf.yaml index 6af9b7d..5ab33ef 100755 --- a/ACDD_conf.yaml +++ b/ACDD_conf.yaml @@ -189,4 +189,4 @@ attrs: parent: !!bool false # CMOR will add a tracking_id if you want to define a prefix add here tracking_id_prefix: - comment: "post-processed using ACCESS-MOPPeR v1.1.0 https://doi.org/10.5281/zenodo.13841181" + comment: "post-processed using ACCESS-MOPPeR v1.1.1 https://doi.org/10.5281/zenodo.14010850" diff --git a/CMIP6_conf.yaml b/CMIP6_conf.yaml index e68c8f0..2deb98e 100755 --- a/CMIP6_conf.yaml +++ b/CMIP6_conf.yaml @@ -180,4 +180,4 @@ attrs: #CMOR will add a tracking_id if you want to define a prefix add here tracking_id_prefix: Conventions: "CF-1.7 CMIP-6.2" - comment: "post-processed using ACCESS-MOPPeR v1.1.0 https://doi.org/10.5281/zenodo.13841181" + comment: "post-processed using ACCESS-MOPPeR v1.1.1 https://doi.org/10.5281/zenodo.14010850" diff --git a/README.md b/README.md index c057ff9..00de0a1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # [ACCESS Model Output Post-Processor (MOPPeR)](https://access-mopper.readthedocs.io/en/latest) [![Read the docs](https://readthedocs.org/projects/access-mopper/badge/?version=latest)](https://access-mopper.readthedocs.io/en/latest/) -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.13841181.svg)](https://doi.org/10.5281/zenodo.13841181) +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.14010850.svg)](https://doi.org/10.5281/zenodo.14010850) This code is derived from the [APP4](https://doi.org/10.5281/zenodo.7703469), initially created by Peter Uhe for CMIP5, and further developed for CMIP6-era by Chloe Mackallah from CSIRO, O&A Aspendale. @@ -11,8 +11,6 @@ It used [CMOR3](https://cmor.llnl.gov/) and files created with the [CMIP6 data r CMOR uses Controlled Vocabularies as metadata constraints, with [CMIP6_CV.json](https://cmor.llnl.gov/mydoc_cmor3_CV/) being the main one. This has an effect on how the data is written in the files, variables' names, directory structure, filenames, and global attributes. The APP4 also relied on mapping files to match the raw model output to CMOR defined variables. To make this approach more flexible we introduced a new tool `mopdb` that helps the users create their own mapping and handling CMOR tables definitions. -Designed for use on ACCESS model output that has been archived using the [ACCESS Archiver tool](https://github.com/ACCESS-Hive/ACCESS-Archiver). - Although we retained a differentiation between `custom` and `cmip` mode the main workflow is the same and `mode` is now only another field in the main configuration file. See [MOPPeR ReadtheDocs](https://access-mopper.readthedocs.io/en/stable/) for the full documentation. diff --git a/conda/meta.yaml b/conda/meta.yaml index af35098..4f8f24c 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,6 +1,6 @@ package: name: mopper - #version: "{{ environ.get('GIT_DESCRIBE_TAG', '1.1.0') }}" + #version: "{{ environ.get('GIT_DESCRIBE_TAG', '1.0') }}" version: '1.1.1' #source: # path: ../.