more fixes of small bugs

docs/calculations.rst

@@ -1,5 +1,5 @@
-Derived variables calculations
-==============================
+Calculating derived variables
+=============================
 
 Calculations are used to derive a variable from one or multiple inputs, to resample a variable to a new frequency or generally to modify a variable so it will match fully the corresponding definition in a CMOR table.
 
@@ -34,6 +34,7 @@ Contributing
 ------------
 TBA
 
+
 Available functions
 -------------------
 
@@ -44,27 +45,31 @@ Atmosphere and aerosol
     :undoc-members:
     :show-inheritance:
 
+
 Ocean
 ^^^^^
 .. automodule:: mopper.calc_ocean
     :members:
     :undoc-members:
     :show-inheritance:
 
+
 SeaIce
 ^^^^^^
 .. automodule:: mopper.calc_seaice
     :members:
     :undoc-members:
     :show-inheritance:
 
+
 Land
 ^^^^
 .. automodule:: mopper.calc_land
     :members:
     :undoc-members:
     :show-inheritance:
 
+
 Other
 ^^^^^
 .. automodule:: mopper.calc_utils

docs/map_exp1.csv

@@ -0,0 +1,27 @@
+cmor_var;input_vars;calculation;units;dimensions;axes;frequency;realm;cell_methods;positive;cmor_table;version;vtype;size;nsteps;fpattern;long_name;standard_name
+amdry;fld_s30i403;;kg m-2;time lat lon;longitude latitude time1;10minPt;atmos;area: time: point;;AUS2200_A10min;AUS2200;float32;22048000;720;umnsa_spec_;TOTAL COLUMN DRY MASS  RHO GRID;
+cl;fld_s00i265;level_to_height(var[0],levs=(0,66));%;time model_theta_level_number lat lon;longitude latitude alevel time1;1hrPt;atmos;area: time: point;;AUS2200_A1hr;AUS2200;float32;1543360000;120;umnsa_cldrad_;AREA CLOUD FRACTION IN EACH LAYER;cloud_area_fraction_in_atmosphere_layer
+cli;fld_s00i012;level_to_height(var[0],levs=(0,66));kg kg-1;time model_theta_level_number lat lon;longitude latitude alevel time1;1hrPt;atmos;area: time: point;;AUS2200_A1hr;AUS2200;float32;1543360000;120;umnsa_cldrad_;QCF AFTER TIMESTEP;mass_fraction_of_cloud_ice_in_air
+eow;fld_s03i232;;kg m-2 s-1;time_0 lat lon;longitude latitude time;1hr;land;area: time: mean;;AUS2200_A1hr;AUS2200;float32;22048000;120;umnsa_slv_;Evaporation flux from open sea;
+rsds;fld_s01i235;;W m-2;time_0 lat lon;longitude latitude time;1hr;atmos;area: time: mean;down;AUS2200_A1hr;AUS2200;float32;22048000;120;umnsa_slv_;TOTAL DOWNWARD SURFACE SW FLUX;surface_downwelling_shortwave_flux_in_air
+...
+# Derived variables with matching version and frequency: Use with caution!;;;;;;;;;;;;;;;;;
+hus24;fld_s00i010 fld_s00i408;plevinterp(var[0], var[1], 24);1;time model_theta_level_number lat lon_0;longitude latitude plev24 time1;1hrPt;atmos;area: time: point;;AUS2200_A1hrPlev;AUS2200;float32;1543360000;120;umnsa_mdl_;SPECIFIC HUMIDITY AFTER TIMESTEP;specific_humidity
+prw;fld_s30i404 fld_s30i403 fld_s30i405 fld_s30i406;var[0]-sum_vars(var[1:]);kg m-2;time lat lon;longitude latitude time1;10minPt;atmos;area: time: point;;AUS2200_A10min;AUS2200;float32;22048000;720;umnsa_spec_;TOTAL COLUMN DRY MASS  RHO GRID;
+...
+# Variables definitions coming from different version;;;;;;;;;;;;;;;;;
+ps;fld_s00i409;;Pa;time_1 lat lon;longitude latitude time1;3hrPt;atmos;area: time: point;;CMIP6_E3hrPt;CM2;float32;22048000;40;umnsaa_pvera;SURFACE PRESSURE AFTER TIMESTEP;surface_air_pressure
+# Variables with different frequency: Use with caution!;;;;;;;;;;;;;;;;;
+ps;fld_s00i409;;Pa;time_1 lat lon;longitude latitude time1;3hrPt;atmos;area: time: point;;CMIP6_E3hrPt;CM2;float32;22048000;40;umnsaa_pvera;SURFACE PRESSURE AFTER TIMESTEP;surface_air_pressure
+...
+# Variables matched using standard_name: complete mapping or discard!;;;;;;;;;;;;;;;;;
+['rsut-AUS2200_A1hr', 'rsutaf-CMIP6_AERmon', 'rsut-CMIP6_Amon', 'rsut-CMIP6_CF3hr', 'rsut-CMIP6_CFday', 'rsut4co2-CMIP6_CFmon', 'rsut-CMIP6_CFsubhr', 'rsut-CMIP6_E1hrClimMon', 'rsut-CMIP6_E3hr', 'rsut-CMIP6_Esubhr'];fld_s01i208;;W m-2;time_4 lat lon;time_4 lat lon;1hrPt;atmos;area: time: point;;;;float32;22048000;120;umnsaa_pvera;OUTGOING SW RAD FLUX (TOA);toa_outgoing_shortwave_flux
+...
+# Derived variables: Use with caution!;;;;;;;;;;;;;;;;;
+prw;fld_s30i404 fld_s30i403 fld_s30i405 fld_s30i406;var[0]-(var[1]+var[2]+var[3]);kg m-2;time lat lon;longitude latitude time;10minPt;atmos;area: time: point;;CMIP6_Amon;CM2;float32;22048000;720;umnsa_spec_;TOTAL COLUMN DRY MASS  RHO GRID;
+zg24;fld_s15i101 fld_s00i408;height_gpheight(var[0], pmod=var[1], levnum=24);m;time model_theta_level_number lat lon_0;longitude latitude plev24 time1;1hrPt;atmos;area: time: point;;AUS2200_A6hr;AUS2200;float32;1543360000;120;umnsa_mdl_;PRESSURE AT THETA LEVELS AFTER TS;air_pressure
+...
+# Variables without mapping;;;;;;;;;;;;;;;;;
+fld_s03i051;fld_s03i051;;;time pseudo_level lat lon;time pseudo_level lat lon;10minPt;atmos;area: time: point;;;;float32;110240000;720;umnsa_spec_;COMBINED RESIST. TO EVAP ON TILES;
+fld_s03i052;fld_s03i052;;;time pseudo_level lat lon;time pseudo_level lat lon;10minPt;atmos;area: time: point;;;;float32;110240000;720;umnsa_spec_;COEFFS RATIO FOR 1.5M T ON TILES;
+...

docs/mopdb_command.rst

@@ -82,33 +82,9 @@ The other groups of records require checking, as either the version or the frequ
 .. _template example:
 .. dropdown:: Example output of template
 
-   cmor_var;input_vars;calculation;units;dimensions;frequency;realm;cell_methods;positive;cmor_table;version;vtype;size;nsteps;filename;long_name;standard_name
-   agesno;fld_s03i832;;day;time pseudo_level_1 lat lon;mon;landIce land;area: time: mean;;CMIP6_LImon;CM2;float32;1880064;12;cw323a.pm;CABLE SNOW AGE ON TILES;age_of_surface_snow
-   amdry;fld_s30i403;;kg m-2;time lat lon;mon;atmos;area: time: mean;;CM2_mon;CM2;float32;110592;12;cw323a.pm;TOTAL COLUMN DRY MASS  RHO GRID;
-   amwet;fld_s30i404;;kg m-2;time lat lon;mon;atmos;area: time: mean;;CM2_mon;CM2;float32;110592;12;cw323a.pm;TOTAL COLUMN WET MASS  RHO GRID;atmosphere_mass_per_unit_area
-   ci;fld_s05i269;;1;time lat lon;mon;atmos;area: time: mean;;CMIP6_Amon;CM2;float32;110592;12;cw323a.pm;deep convection indicator;
-   ...
-   # Derived variables with matching version and frequency: Use with caution!;;;;;;;;;;;;;;;;
-   baresoilFrac;fld_s03i317 fld_s03i395;extract_tilefrac(var[0],14,landfrac=var[1]);1;time pseudo_level_1 lat lon;mon;land;area: time: mean;;CMIP6_Lmon;CM2;float32;1880064;12;cw323a.pm;SURFACE TILE FRACTIONS;
-   c3PftFrac;fld_s03i317 fld_s03i395;extract_tilefrac(var[0],[1,2,3,4,5,6,8,9,11],landfrac=var[1]);1;time pseudo_level_1 lat lon;mon;land;area: time: mean;;CMIP6_Lmon;CM2;float32;1880064;12;cw323a.pm;SURFACE TILE FRACTIONS; 
-   # Variables definitions coming from different version;;;;;;;;;;;;;;;;
-   rlntds;fld_s02i203;;W m-2;time lat lon;mon;ocean;area: time: mean; time: mean;;CMIP6_Omon;float32;110592;12;cw323a.pm;NET DN LW RAD FLUX:OPEN SEA:SEA MEAN;surface_net_downward_longwave_flux
-   rssntds;fld_s01i203;;W m-2;time lat lon;mon;ocean;area: time: mean; time: mean;;CM2_mon;float32;110592;12;cw323a.pm;NET DN SW RAD FLUX:OPEN SEA:SEA MEAN;surface_net_downward_shortwave_flux
-   # Variables with different frequency: Use with caution!;;;;;;;;;;;;;;;;
-   rlntds;fld_s02i203;;W m-2;time lat lon;mon;ocean;area: time: mean; time: mean;;CMIP6_Omon;float32;110592;12;cw323a.pm;NET DN LW RAD FLUX:OPEN SEA:SEA MEAN;surface_net_downward_longwave_flux
-   rssntds;fld_s01i203;;W m-2;time lat lon;mon;ocean;area: time: mean; time: mean;;CM2_mon;float32;110592;12;cw323a.pm;NET DN SW RAD FLUX:OPEN SEA:SEA MEAN;surface_net_downward_shortwave_flux
-   # Variables matched using standard_name: Use with caution!;;;;;;;;;;;;;;;;
-   ['huss-CMIP6_3hr', 'hus-CMIP6_6hrLev', 'hus4-CMIP6_6hrPlev', 'hus27-CMIP6_6hrPlevPt', 'hus7h-CMIP6_6hrPlevPt', 'huss-CMIP6_6hrPlevPt', 'hus-CMIP6_Amon', 'huss-CMIP6_Amon', 'hus-CMIP6_CFday', 'hus-CMIP6_CFmon', 'hus-CMIP6_CFsubhr', 'huss-CMIP6_CFsubhr', 'hus-CMIP6_day', 'huss-CMIP6_day', 'hus-CMIP6_E3hrPt', 'hus7h-CMIP6_E3hrPt', 'hus-CMIP6_Eday', 'hus850-CMIP6_Eday', 'hus-CMIP6_EdayZ', 'hus-CMIP6_Emon', 'hus27-CMIP6_Emon', 'hussLut-CMIP6_Emon', 'hus-CMIP6_Esubhr', 'huss-CMIP6_Esubhr', 'huss-AUS2200_A10min', 'hus-AUS2200_A1hr', 'huss-AUS2200_A1hr', 'hus24-AUS2200_A1hrPlev', 'hus3-AUS2200_A1hrPlev'];;;1;time model_theta_level_number lat lon;mon;;area: time: mean;;CMIP6_Amon;;float32;9400320;12;cw323a.pm;SPECIFIC HUMIDITY AFTER TIMESTEP;specific_humidity 
-   ...
-   # Derived variables: Use with caution!;;;;;;;;;;;;;;;;
-   hus24;fld_s00i010 fld_s00i408;plevinterp(var[0], var[1], 24);1;time model_theta_level_number lat lon;mon;atmos;area: time: mean;;AUS2200_A1hrPlev;AUS2200;float32;9400320;12;cw323a.pm;SPECIFIC HUMIDITY AFTER TIMESTEP;specific_humidity
-   sifllatstop;fld_s03i234 fld_s00i031;maskSeaIce(var[0],var[1]);1;time lat lon;mon;seaIce;area: time: mean;up;AUS2200_A1hr;AUS2200;float32;110592;12;cw323a.pm;FRAC OF SEA ICE IN SEA AFTER TSTEP;sea_ice_area_fraction
-   theta24;fld_s00i004 fld_s00i408;plevinterp(var[0], var[1], 24);K;time model_theta_level_number lat lon;mon;atmos;area: time: mean;;AUS2200_A1hrPlev;AUS2200;float32;9400320;12;cw323a.pm;THETA AFTER TIMESTEP;air_potential_temperature
-   # Variables without mapping;;;;;;;;;;;;;;;;
-   fld_s00i211;;;1;time model_theta_level_number lat lon;mon;;area: time: mean;;;;float32;9400320;12;cw323a.pm;Convective cloud amount with anvil;
-   fld_s00i253;;;;time model_rho_level_number lat lon;mon;;area: time: mean;;;;float32;9400320;12;cw323a.pm;DENSITY*R*R AFTER TIMESTEP;
-   fld_s00i413;;;1;time pseudo_level lat lon;mon;;area: time: mean;;;;float32;552960;12;cw323a.pm;Sea ice concentration by categories;
-   ...
+   .. literalinclude:: map_ex1.csv
+      :language: csv
+
 
 
 Create an intake catalogue

docs/output.rst

@@ -33,6 +33,7 @@ This folder will contain the following files:
     * frequency - output variable frequency
     * realm - output variable realm
     * timeshot - cell_methods value for time: point, mean, sum, max, min
+    * axes - The cmor names of the axes used in variable definition
     * tstart - datetime stamp for time range start
     * tend - datetime stamp for time range end
     * sel_start - datetime stamp to use for input file selection (start)
@@ -72,10 +73,11 @@ This folder will contain the following files:
     | # for a list of packages
     |
     | module use /g/data/hh5/public/modules
-    | module load conda/analysis3-23.04
+    | module load conda/analysis3
+    | source mopper_env/bin/activate # if using conda option
     |
     | cd /g/data/ua8/Working/packages/ACCESS-MOPPeR
-    | mopper.py  -i ashwed1980_config.yaml run
+    | mop  run -c ashwed1980_config.yaml # --debug (uncomment to run in debug mode)
     | echo 'APP completed for exp ashwed1980.'
 
 * **experiment-id.json**

mappings/map_atmos_CM2.csv

@@ -27,6 +27,7 @@ hur19uvgrid;fld_s30i206;;%;time pressure lat_v lon_u;longitude latitude plev19 t
 hus;fld_s00i010;level_to_height(var[0]);1;time model_theta_level_number lat lon;longitude latitude alevel time;mon;atmos;area: time: mean;;CMIP6_CFmon;CM2;float32;9400320;12;cw323a.pm;SPECIFIC HUMIDITY AFTER TIMESTEP;specific_humidity
 hus;fld_s30i295;;1;time pressure lat lon;longitude latitude plev19 time;mon;atmos;area: time: mean;;CMIP6_Amon;CM2;float32;2101248;12;cw323a.pm;SPECIFIC HUMIDITY ON P LEV/T GRID;specific_humidity
 hus;fld_s30i295;;1;time pressure lat lon;longitude latitude plev19 time;day;atmos;area: time: mean;;CMIP6_Eday;CM2;float32;884736;74772;cm000a.pd;SPECIFIC HUMIDITY ON P LEV/T GRID;specific_humidity
+hus;fld_s30i295;;1;time pressure lat lon;longitude latitude plev8 time;day;atmos;area: time: mean;;CMIP6_day;CM2;float32;884736;74772;cm000a.pd;SPECIFIC HUMIDITY ON P LEV/T GRID;specific_humidity
 huss;fld_s03i237;;1;time lat lon;longitude latitude time height1.5m;mon;atmos;area: time: mean;;CM2_mon;CM2;float32;110592;12;cw323a.pm;SPECIFIC HUMIDITY  AT 1.5M;specific_humidity
 huss;fld_s03i237;;1;time lat lon;longitude latitude time height1.5m;3hrPt;atmos;area: time: point;;CM2_3hr;CM2;float32;110592;578880;cm000a.p8;SPECIFIC HUMIDITY  AT 1.5M;specific_humidity
 huss;fld_s03i237;;1;time lat lon;longitude latitude time height1.5m;day;atmos;area: time: mean;;CM2_day;CM2;float32;110592;74772;cm000a.pd;SPECIFIC HUMIDITY  AT 1.5M;specific_humidity

mappings/map_atmos_ESM1.5.csv

@@ -27,6 +27,7 @@ hur19uvgrid;fld_s30i206;;%;time pressure lat_v lon_u;longitude latitude plev19 t
 hus;fld_s00i010;level_to_height(var[0]);1;time model_theta_level_number lat lon;longitude latitude alevel time;mon;atmos;area: time: mean;;CMIP6_CFmon;ESM1.5;float32;9400320;12;cw323a.pm;SPECIFIC HUMIDITY AFTER TIMESTEP;specific_humidity
 hus;fld_s30i295;;1;time pressure lat lon;longitude latitude plev19 time;mon;atmos;area: time: mean;;CMIP6_Amon;ESM1.5;float32;2101248;12;cw323a.pm;SPECIFIC HUMIDITY ON P LEV/T GRID;specific_humidity
 hus;fld_s30i295;;1;time pressure lat lon;longitude latitude plev19 time;day;atmos;area: time: mean;;CMIP6_Eday;ESM1.5;float32;884736;74772;cm000a.pd;SPECIFIC HUMIDITY ON P LEV/T GRID;specific_humidity
+hus;fld_s30i295;;1;time pressure lat lon;longitude latitude plev8 time;day;atmos;area: time: mean;;CMIP6_day;ESM1.5;float32;884736;74772;cm000a.pd;SPECIFIC HUMIDITY ON P LEV/T GRID;specific_humidity
 huss;fld_s03i237;;1;time lat lon;longitude latitude time height1.5m;mon;atmos;area: time: mean;;CM2_mon;ESM1.5;float32;110592;12;cw323a.pm;SPECIFIC HUMIDITY  AT 1.5M;specific_humidity
 huss;fld_s03i237;;1;time lat lon;longitude latitude time height1.5m;3hrPt;atmos;area: time: point;;CM2_3hr;ESM1.5;float32;110592;578880;cm000a.p8;SPECIFIC HUMIDITY  AT 1.5M;specific_humidity
 huss;fld_s03i237;;1;time lat lon;longitude latitude time height1.5m;day;atmos;area: time: mean;;CM2_day;ESM1.5;float32;110592;74772;cm000a.pd;SPECIFIC HUMIDITY  AT 1.5M;specific_humidity