From 3bf034b39fab555c1cca04b17413c5c7bc7399f3 Mon Sep 17 00:00:00 2001 From: Gabriel Moldovan Date: Fri, 2 Aug 2024 10:38:07 +0000 Subject: [PATCH 01/21] added config for bounding --- src/anemoi/training/config/data/zarr.yaml | 7 +++++++ src/anemoi/training/config/training/default.yaml | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/anemoi/training/config/data/zarr.yaml b/src/anemoi/training/config/data/zarr.yaml index 27c17edb..f77fef76 100644 --- a/src/anemoi/training/config/data/zarr.yaml +++ b/src/anemoi/training/config/data/zarr.yaml @@ -30,6 +30,9 @@ diagnostic: normalizer: default: "mean-std" min-max: + # std: + # - "tp" + # - "cp" max: - "sdor" - "slor" @@ -49,6 +52,10 @@ normalizer: imputer: default: "none" + # # For the fraction activation to be robust, the fraction variable should + # # be normalised with the standard deviation of the total variable + # fraction_normalisation: [["tp","cp"]] #first variable is the total variable + # processors including imputers and normalizers are applied in order of definition processors: # example_imputer: diff --git a/src/anemoi/training/config/training/default.yaml b/src/anemoi/training/config/training/default.yaml index 870eeb7a..815a1d78 100644 --- a/src/anemoi/training/config/training/default.yaml +++ b/src/anemoi/training/config/training/default.yaml @@ -79,6 +79,16 @@ metrics: - u_850 - v_850 +# Bounding configuration +# bounding_strategies: #order in which these variables are defined matters! +# tp: +# _target_: anemoi.models.models.bounding.ReluBoundingStrategy #[0, infinity) +# cp: +# _target_: anemoi.models.models.bounding.FractionHardtanhBoundingStrategy # fraction of tp +# min_val: 0 +# max_val: 1 +# total_var: tp + pressure_level_scaler: _target_: anemoi.training.data.scaling.ReluPressureLevelScaler minimum: 0.2 From 8b86e78a51dcd334e06ccd3827d84f438d380021 Mon Sep 17 00:00:00 2001 From: Jesper Dramsch Date: Wed, 28 Aug 2024 14:56:40 +0000 Subject: [PATCH 02/21] refactor: move bounding config to model --- src/anemoi/training/config/model/gnn.yaml | 12 ++++++++++++ .../training/config/model/graphtransformer.yaml | 13 +++++++++++++ src/anemoi/training/config/model/transformer.yaml | 13 +++++++++++++ src/anemoi/training/config/training/default.yaml | 10 ---------- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/src/anemoi/training/config/model/gnn.yaml b/src/anemoi/training/config/model/gnn.yaml index 8e3f39ae..407195cc 100644 --- a/src/anemoi/training/config/model/gnn.yaml +++ b/src/anemoi/training/config/model/gnn.yaml @@ -43,3 +43,15 @@ attributes: nodes: [] node_loss_weight: area_weight + +# Bounding configuration +# bounding: #These are applied in order +# - _target_: anemoi.models.layers.bounding.ReluBoundingStrategy #[0, infinity) +# variables: +# - tp +# - _target_: anemoi.models.layers.bounding.FractionHardtanhBoundingStrategy # fraction of tp +# variables: +# - cp +# min_val: 0 +# max_val: 1 +# total_var: tp diff --git a/src/anemoi/training/config/model/graphtransformer.yaml b/src/anemoi/training/config/model/graphtransformer.yaml index 622d3ae3..8637bb4d 100644 --- a/src/anemoi/training/config/model/graphtransformer.yaml +++ b/src/anemoi/training/config/model/graphtransformer.yaml @@ -48,3 +48,16 @@ attributes: nodes: [] node_loss_weight: area_weight + + +# Bounding configuration +# bounding: #These are applied in order +# - _target_: anemoi.models.layers.bounding.ReluBoundingStrategy #[0, infinity) +# variables: +# - tp +# - _target_: anemoi.models.layers.bounding.FractionHardtanhBoundingStrategy # fraction of tp +# variables: +# - cp +# min_val: 0 +# max_val: 1 +# total_var: tp diff --git a/src/anemoi/training/config/model/transformer.yaml b/src/anemoi/training/config/model/transformer.yaml index ae8257ab..868eb08f 100644 --- a/src/anemoi/training/config/model/transformer.yaml +++ b/src/anemoi/training/config/model/transformer.yaml @@ -47,3 +47,16 @@ attributes: nodes: [] node_loss_weight: area_weight + + +# Bounding configuration +# bounding: #These are applied in order +# - _target_: anemoi.models.layers.bounding.ReluBoundingStrategy #[0, infinity) +# variables: +# - tp +# - _target_: anemoi.models.layers.bounding.FractionHardtanhBoundingStrategy # fraction of tp +# variables: +# - cp +# min_val: 0 +# max_val: 1 +# total_var: tp diff --git a/src/anemoi/training/config/training/default.yaml b/src/anemoi/training/config/training/default.yaml index 815a1d78..870eeb7a 100644 --- a/src/anemoi/training/config/training/default.yaml +++ b/src/anemoi/training/config/training/default.yaml @@ -79,16 +79,6 @@ metrics: - u_850 - v_850 -# Bounding configuration -# bounding_strategies: #order in which these variables are defined matters! -# tp: -# _target_: anemoi.models.models.bounding.ReluBoundingStrategy #[0, infinity) -# cp: -# _target_: anemoi.models.models.bounding.FractionHardtanhBoundingStrategy # fraction of tp -# min_val: 0 -# max_val: 1 -# total_var: tp - pressure_level_scaler: _target_: anemoi.training.data.scaling.ReluPressureLevelScaler minimum: 0.2 From b52b06cff18164e1841f2df53788c34efca39958 Mon Sep 17 00:00:00 2001 From: Gabriel Moldovan Date: Fri, 30 Aug 2024 10:05:32 +0000 Subject: [PATCH 03/21] fixed typo in bounding functions config --- src/anemoi/training/config/model/gnn.yaml | 4 ++-- src/anemoi/training/config/model/graphtransformer.yaml | 5 ++--- src/anemoi/training/config/model/transformer.yaml | 5 ++--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/anemoi/training/config/model/gnn.yaml b/src/anemoi/training/config/model/gnn.yaml index 407195cc..5745881e 100644 --- a/src/anemoi/training/config/model/gnn.yaml +++ b/src/anemoi/training/config/model/gnn.yaml @@ -46,10 +46,10 @@ node_loss_weight: area_weight # Bounding configuration # bounding: #These are applied in order -# - _target_: anemoi.models.layers.bounding.ReluBoundingStrategy #[0, infinity) +# - _target_: anemoi.models.layers.bounding.ReluBounding #[0, infinity) # variables: # - tp -# - _target_: anemoi.models.layers.bounding.FractionHardtanhBoundingStrategy # fraction of tp +# - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp # variables: # - cp # min_val: 0 diff --git a/src/anemoi/training/config/model/graphtransformer.yaml b/src/anemoi/training/config/model/graphtransformer.yaml index 8637bb4d..b71e8eea 100644 --- a/src/anemoi/training/config/model/graphtransformer.yaml +++ b/src/anemoi/training/config/model/graphtransformer.yaml @@ -49,13 +49,12 @@ attributes: node_loss_weight: area_weight - # Bounding configuration # bounding: #These are applied in order -# - _target_: anemoi.models.layers.bounding.ReluBoundingStrategy #[0, infinity) +# - _target_: anemoi.models.layers.bounding.ReluBounding #[0, infinity) # variables: # - tp -# - _target_: anemoi.models.layers.bounding.FractionHardtanhBoundingStrategy # fraction of tp +# - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp # variables: # - cp # min_val: 0 diff --git a/src/anemoi/training/config/model/transformer.yaml b/src/anemoi/training/config/model/transformer.yaml index 868eb08f..52f17c7c 100644 --- a/src/anemoi/training/config/model/transformer.yaml +++ b/src/anemoi/training/config/model/transformer.yaml @@ -48,13 +48,12 @@ attributes: node_loss_weight: area_weight - # Bounding configuration # bounding: #These are applied in order -# - _target_: anemoi.models.layers.bounding.ReluBoundingStrategy #[0, infinity) +# - _target_: anemoi.models.layers.bounding.ReluBounding #[0, infinity) # variables: # - tp -# - _target_: anemoi.models.layers.bounding.FractionHardtanhBoundingStrategy # fraction of tp +# - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp # variables: # - cp # min_val: 0 From 81d450c82fabfaa36b7398fe8d6535976d74d158 Mon Sep 17 00:00:00 2001 From: Jesper Dramsch Date: Mon, 23 Sep 2024 13:12:09 +0000 Subject: [PATCH 04/21] docs: changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index de7b1209..bc5b3d58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ Keep it human-readable, your future self will thank you! ### Added +- Variable Bounding as configurable model layers [#13](https://github.com/ecmwf/anemoi-models/issues/13) + #### Functionality - Enable the callback for plotting a histogram for variables containing NaNs From 390ae3739fa600689db86e3a19c7e8d809cc81f8 Mon Sep 17 00:00:00 2001 From: gabrieloks <116646686+gabrieloks@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:04:24 +0100 Subject: [PATCH 05/21] Update zarr.yaml --- src/anemoi/training/config/data/zarr.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/anemoi/training/config/data/zarr.yaml b/src/anemoi/training/config/data/zarr.yaml index 0f45d944..2b77536b 100644 --- a/src/anemoi/training/config/data/zarr.yaml +++ b/src/anemoi/training/config/data/zarr.yaml @@ -30,10 +30,12 @@ remapped: normalizer: default: "mean-std" + remap: + cp: tp + std: + - "tp" + - "cp" min-max: - # std: - # - "tp" - # - "cp" max: - "sdor" - "slor" From 3da5adc6044dabc2a9e1bf6f88b68768f728350f Mon Sep 17 00:00:00 2001 From: gabrieloks <116646686+gabrieloks@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:05:58 +0100 Subject: [PATCH 06/21] Update zarr.yaml --- src/anemoi/training/config/data/zarr.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/anemoi/training/config/data/zarr.yaml b/src/anemoi/training/config/data/zarr.yaml index 2b77536b..efede712 100644 --- a/src/anemoi/training/config/data/zarr.yaml +++ b/src/anemoi/training/config/data/zarr.yaml @@ -57,10 +57,6 @@ imputer: remapper: default: "none" - # # For the fraction activation to be robust, the fraction variable should - # # be normalised with the standard deviation of the total variable - # fraction_normalisation: [["tp","cp"]] #first variable is the total variable - # processors including imputers and normalizers are applied in order of definition processors: # example_imputer: From a085ad5290303f550aed54fe362319c6b7fb6920 Mon Sep 17 00:00:00 2001 From: gabrieloks <116646686+gabrieloks@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:06:53 +0100 Subject: [PATCH 07/21] Update gnn.yaml --- src/anemoi/training/config/model/gnn.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/anemoi/training/config/model/gnn.yaml b/src/anemoi/training/config/model/gnn.yaml index d06a848f..b3b5993e 100644 --- a/src/anemoi/training/config/model/gnn.yaml +++ b/src/anemoi/training/config/model/gnn.yaml @@ -48,13 +48,13 @@ attributes: node_loss_weight: area_weight # Bounding configuration -# bounding: #These are applied in order -# - _target_: anemoi.models.layers.bounding.ReluBounding #[0, infinity) -# variables: -# - tp -# - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp -# variables: -# - cp -# min_val: 0 -# max_val: 1 -# total_var: tp +bounding: #These are applied in order + - _target_: anemoi.models.layers.bounding.ReluBounding #[0, infinity) + variables: + - tp + - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp + variables: + - cp + min_val: 0 + max_val: 1 + total_var: tp From 2fd627eda899acaf63c46f682d99bfc7b1a333dc Mon Sep 17 00:00:00 2001 From: gabrieloks <116646686+gabrieloks@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:07:33 +0100 Subject: [PATCH 08/21] Update graphtransformer.yaml --- .../config/model/graphtransformer.yaml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/anemoi/training/config/model/graphtransformer.yaml b/src/anemoi/training/config/model/graphtransformer.yaml index 830ccefe..6ed7686c 100644 --- a/src/anemoi/training/config/model/graphtransformer.yaml +++ b/src/anemoi/training/config/model/graphtransformer.yaml @@ -53,13 +53,13 @@ attributes: node_loss_weight: area_weight # Bounding configuration -# bounding: #These are applied in order -# - _target_: anemoi.models.layers.bounding.ReluBounding #[0, infinity) -# variables: -# - tp -# - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp -# variables: -# - cp -# min_val: 0 -# max_val: 1 -# total_var: tp +bounding: #These are applied in order + - _target_: anemoi.models.layers.bounding.ReluBounding #[0, infinity) + variables: + - tp + - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp + variables: + - cp + min_val: 0 + max_val: 1 + total_var: tp From 9250c0e6b22a6a6e4c1d2668bcbe009cdf70644b Mon Sep 17 00:00:00 2001 From: gabrieloks <116646686+gabrieloks@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:07:53 +0100 Subject: [PATCH 09/21] Update transformer.yaml --- .../training/config/model/transformer.yaml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/anemoi/training/config/model/transformer.yaml b/src/anemoi/training/config/model/transformer.yaml index 6469241b..02ca37ca 100644 --- a/src/anemoi/training/config/model/transformer.yaml +++ b/src/anemoi/training/config/model/transformer.yaml @@ -52,13 +52,13 @@ attributes: node_loss_weight: area_weight # Bounding configuration -# bounding: #These are applied in order -# - _target_: anemoi.models.layers.bounding.ReluBounding #[0, infinity) -# variables: -# - tp -# - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp -# variables: -# - cp -# min_val: 0 -# max_val: 1 -# total_var: tp +bounding: #These are applied in order + - _target_: anemoi.models.layers.bounding.ReluBounding #[0, infinity) + variables: + - tp + - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp + variables: + - cp + min_val: 0 + max_val: 1 + total_var: tp From f81ee097ba6a5484e62ff904ebcd5c5ae3873e75 Mon Sep 17 00:00:00 2001 From: gabrieloks <116646686+gabrieloks@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:07:44 +0000 Subject: [PATCH 10/21] Update gnn.yaml --- src/anemoi/training/config/model/gnn.yaml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/anemoi/training/config/model/gnn.yaml b/src/anemoi/training/config/model/gnn.yaml index b3b5993e..513b6bd8 100644 --- a/src/anemoi/training/config/model/gnn.yaml +++ b/src/anemoi/training/config/model/gnn.yaml @@ -49,12 +49,20 @@ node_loss_weight: area_weight # Bounding configuration bounding: #These are applied in order + + # Bound tp (total precipitation) with a Relu bounding layer + # ensuring a range of [0, infinity) to avoid negative precipitation values. - _target_: anemoi.models.layers.bounding.ReluBounding #[0, infinity) variables: - tp - - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp - variables: - - cp - min_val: 0 - max_val: 1 - total_var: tp + + # [OPTIONAL] Bound cp (convective precipitation) as a fraction of tp. + # This guarantees that cp is physically consistent with tp by restricting cp + # to a fraction of tp [0 to 1]. Uncomment the lines below to apply. + + # - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp + # variables: + # - cp + # min_val: 0 + # max_val: 1 + # total_var: tp From b871132aafc08134b4926bd152a736b15988dce5 Mon Sep 17 00:00:00 2001 From: gabrieloks <116646686+gabrieloks@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:08:14 +0000 Subject: [PATCH 11/21] Update graphtransformer.yaml --- .../config/model/graphtransformer.yaml | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/anemoi/training/config/model/graphtransformer.yaml b/src/anemoi/training/config/model/graphtransformer.yaml index 6ed7686c..e28b62b3 100644 --- a/src/anemoi/training/config/model/graphtransformer.yaml +++ b/src/anemoi/training/config/model/graphtransformer.yaml @@ -54,12 +54,20 @@ node_loss_weight: area_weight # Bounding configuration bounding: #These are applied in order + + # Bound tp (total precipitation) with a Relu bounding layer + # ensuring a range of [0, infinity) to avoid negative precipitation values. - _target_: anemoi.models.layers.bounding.ReluBounding #[0, infinity) variables: - tp - - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp - variables: - - cp - min_val: 0 - max_val: 1 - total_var: tp + + # [OPTIONAL] Bound cp (convective precipitation) as a fraction of tp. + # This guarantees that cp is physically consistent with tp by restricting cp + # to a fraction of tp [0 to 1]. Uncomment the lines below to apply. + + # - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp + # variables: + # - cp + # min_val: 0 + # max_val: 1 + # total_var: tp From a29bf91a54c9b3050b5b553351fcb91e9300fee0 Mon Sep 17 00:00:00 2001 From: gabrieloks <116646686+gabrieloks@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:08:41 +0000 Subject: [PATCH 12/21] Update transformer.yaml --- .../training/config/model/transformer.yaml | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/anemoi/training/config/model/transformer.yaml b/src/anemoi/training/config/model/transformer.yaml index 02ca37ca..fb08dd71 100644 --- a/src/anemoi/training/config/model/transformer.yaml +++ b/src/anemoi/training/config/model/transformer.yaml @@ -53,12 +53,20 @@ node_loss_weight: area_weight # Bounding configuration bounding: #These are applied in order + + # Bound tp (total precipitation) with a Relu bounding layer + # ensuring a range of [0, infinity) to avoid negative precipitation values. - _target_: anemoi.models.layers.bounding.ReluBounding #[0, infinity) variables: - tp - - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp - variables: - - cp - min_val: 0 - max_val: 1 - total_var: tp + + # [OPTIONAL] Bound cp (convective precipitation) as a fraction of tp. + # This guarantees that cp is physically consistent with tp by restricting cp + # to a fraction of tp [0 to 1]. Uncomment the lines below to apply. + + # - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp + # variables: + # - cp + # min_val: 0 + # max_val: 1 + # total_var: tp From 8135bd1863db84b8f72338a75c1ea5ec3dc4cf87 Mon Sep 17 00:00:00 2001 From: gabrieloks <116646686+gabrieloks@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:16:00 +0000 Subject: [PATCH 13/21] Update transformer.yaml --- src/anemoi/training/config/model/transformer.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/anemoi/training/config/model/transformer.yaml b/src/anemoi/training/config/model/transformer.yaml index fb08dd71..0a835d12 100644 --- a/src/anemoi/training/config/model/transformer.yaml +++ b/src/anemoi/training/config/model/transformer.yaml @@ -62,7 +62,10 @@ bounding: #These are applied in order # [OPTIONAL] Bound cp (convective precipitation) as a fraction of tp. # This guarantees that cp is physically consistent with tp by restricting cp - # to a fraction of tp [0 to 1]. Uncomment the lines below to apply. + # to a fraction of tp [0 to 1]. Uncomment the lines below to apply. + # NOTE: If this bounding strategy is used, the normalization of cp must be + # changed to "std" normalization, and the "cp" statistics should be remapped + # to those of tp to ensure consistency. # - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp # variables: From d16821214d3f92220c31807b6fd64c160bb236a0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:16:11 +0000 Subject: [PATCH 14/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/anemoi/training/config/model/transformer.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/anemoi/training/config/model/transformer.yaml b/src/anemoi/training/config/model/transformer.yaml index 0a835d12..b26c9ecc 100644 --- a/src/anemoi/training/config/model/transformer.yaml +++ b/src/anemoi/training/config/model/transformer.yaml @@ -62,9 +62,9 @@ bounding: #These are applied in order # [OPTIONAL] Bound cp (convective precipitation) as a fraction of tp. # This guarantees that cp is physically consistent with tp by restricting cp - # to a fraction of tp [0 to 1]. Uncomment the lines below to apply. - # NOTE: If this bounding strategy is used, the normalization of cp must be - # changed to "std" normalization, and the "cp" statistics should be remapped + # to a fraction of tp [0 to 1]. Uncomment the lines below to apply. + # NOTE: If this bounding strategy is used, the normalization of cp must be + # changed to "std" normalization, and the "cp" statistics should be remapped # to those of tp to ensure consistency. # - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp From 125c6d1ca6719a5f428f8d796c3ee7c2c287c08a Mon Sep 17 00:00:00 2001 From: gabrieloks <116646686+gabrieloks@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:16:20 +0000 Subject: [PATCH 15/21] Update graphtransformer.yaml --- src/anemoi/training/config/model/graphtransformer.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/anemoi/training/config/model/graphtransformer.yaml b/src/anemoi/training/config/model/graphtransformer.yaml index e28b62b3..67c08a58 100644 --- a/src/anemoi/training/config/model/graphtransformer.yaml +++ b/src/anemoi/training/config/model/graphtransformer.yaml @@ -63,7 +63,10 @@ bounding: #These are applied in order # [OPTIONAL] Bound cp (convective precipitation) as a fraction of tp. # This guarantees that cp is physically consistent with tp by restricting cp - # to a fraction of tp [0 to 1]. Uncomment the lines below to apply. + # to a fraction of tp [0 to 1]. Uncomment the lines below to apply. + # NOTE: If this bounding strategy is used, the normalization of cp must be + # changed to "std" normalization, and the "cp" statistics should be remapped + # to those of tp to ensure consistency. # - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp # variables: From 7028c9db6568160599975c28a78b92c2c97076cc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:16:35 +0000 Subject: [PATCH 16/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/anemoi/training/config/model/graphtransformer.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/anemoi/training/config/model/graphtransformer.yaml b/src/anemoi/training/config/model/graphtransformer.yaml index 67c08a58..5c2e819a 100644 --- a/src/anemoi/training/config/model/graphtransformer.yaml +++ b/src/anemoi/training/config/model/graphtransformer.yaml @@ -63,9 +63,9 @@ bounding: #These are applied in order # [OPTIONAL] Bound cp (convective precipitation) as a fraction of tp. # This guarantees that cp is physically consistent with tp by restricting cp - # to a fraction of tp [0 to 1]. Uncomment the lines below to apply. - # NOTE: If this bounding strategy is used, the normalization of cp must be - # changed to "std" normalization, and the "cp" statistics should be remapped + # to a fraction of tp [0 to 1]. Uncomment the lines below to apply. + # NOTE: If this bounding strategy is used, the normalization of cp must be + # changed to "std" normalization, and the "cp" statistics should be remapped # to those of tp to ensure consistency. # - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp From 958941c10542c9784d4133ed49b768f6fc108c5e Mon Sep 17 00:00:00 2001 From: gabrieloks <116646686+gabrieloks@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:17:33 +0000 Subject: [PATCH 17/21] Update gnn.yaml --- src/anemoi/training/config/model/gnn.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/anemoi/training/config/model/gnn.yaml b/src/anemoi/training/config/model/gnn.yaml index 513b6bd8..baf498cc 100644 --- a/src/anemoi/training/config/model/gnn.yaml +++ b/src/anemoi/training/config/model/gnn.yaml @@ -58,7 +58,10 @@ bounding: #These are applied in order # [OPTIONAL] Bound cp (convective precipitation) as a fraction of tp. # This guarantees that cp is physically consistent with tp by restricting cp - # to a fraction of tp [0 to 1]. Uncomment the lines below to apply. + # to a fraction of tp [0 to 1]. Uncomment the lines below to apply. + # NOTE: If this bounding strategy is used, the normalization of cp must be + # changed to "std" normalization, and the "cp" statistics should be remapped + # to those of tp to ensure consistency. # - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp # variables: From 6aac75458dae95d786f9723c68f1e9b83a169108 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:17:45 +0000 Subject: [PATCH 18/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/anemoi/training/config/model/gnn.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/anemoi/training/config/model/gnn.yaml b/src/anemoi/training/config/model/gnn.yaml index baf498cc..4f4c176c 100644 --- a/src/anemoi/training/config/model/gnn.yaml +++ b/src/anemoi/training/config/model/gnn.yaml @@ -58,9 +58,9 @@ bounding: #These are applied in order # [OPTIONAL] Bound cp (convective precipitation) as a fraction of tp. # This guarantees that cp is physically consistent with tp by restricting cp - # to a fraction of tp [0 to 1]. Uncomment the lines below to apply. - # NOTE: If this bounding strategy is used, the normalization of cp must be - # changed to "std" normalization, and the "cp" statistics should be remapped + # to a fraction of tp [0 to 1]. Uncomment the lines below to apply. + # NOTE: If this bounding strategy is used, the normalization of cp must be + # changed to "std" normalization, and the "cp" statistics should be remapped # to those of tp to ensure consistency. # - _target_: anemoi.models.layers.bounding.FractionBounding # fraction of tp From 5b39ab47d91ecd33a1de266e40d181cc48630374 Mon Sep 17 00:00:00 2001 From: gabrieloks <116646686+gabrieloks@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:23:04 +0000 Subject: [PATCH 19/21] Update zarr.yaml --- src/anemoi/training/config/data/zarr.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/anemoi/training/config/data/zarr.yaml b/src/anemoi/training/config/data/zarr.yaml index efede712..e3ccdd43 100644 --- a/src/anemoi/training/config/data/zarr.yaml +++ b/src/anemoi/training/config/data/zarr.yaml @@ -30,11 +30,21 @@ remapped: normalizer: default: "mean-std" - remap: - cp: tp + + # Remap cp statistics to those of tp when using FractionBounding. This ensures + # that cp, as a fraction of tp, remains consistent with tp's scale and statistics. + # NOTE: This remap should only be applied if FractionBounding is enabled for cp. + # remap: + # cp: tp + + # Standardization applied to tp and cp variables. Ensure that if cp is bounded + # as a fraction of tp, both variables are normalized using these shared statistics. + # "Std" normalization is preferred here over "mean-std" to avoid shifting of the + # zero value in the normalized space. std: - "tp" - "cp" + min-max: max: - "sdor" From b686937fff9f73946a871ac20c5d8937ec299722 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:23:17 +0000 Subject: [PATCH 20/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/anemoi/training/config/data/zarr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/anemoi/training/config/data/zarr.yaml b/src/anemoi/training/config/data/zarr.yaml index e3ccdd43..18634c5e 100644 --- a/src/anemoi/training/config/data/zarr.yaml +++ b/src/anemoi/training/config/data/zarr.yaml @@ -37,9 +37,9 @@ normalizer: # remap: # cp: tp - # Standardization applied to tp and cp variables. Ensure that if cp is bounded + # Standardization applied to tp and cp variables. Ensure that if cp is bounded # as a fraction of tp, both variables are normalized using these shared statistics. - # "Std" normalization is preferred here over "mean-std" to avoid shifting of the + # "Std" normalization is preferred here over "mean-std" to avoid shifting of the # zero value in the normalized space. std: - "tp" From 70ae43072abd333488b58b7f355cf5b3055a7571 Mon Sep 17 00:00:00 2001 From: gabrieloks <116646686+gabrieloks@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:24:01 +0000 Subject: [PATCH 21/21] Update zarr.yaml --- src/anemoi/training/config/data/zarr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/anemoi/training/config/data/zarr.yaml b/src/anemoi/training/config/data/zarr.yaml index 18634c5e..3b9a4537 100644 --- a/src/anemoi/training/config/data/zarr.yaml +++ b/src/anemoi/training/config/data/zarr.yaml @@ -43,7 +43,7 @@ normalizer: # zero value in the normalized space. std: - "tp" - - "cp" + # - "cp" min-max: max: