From 2a59ffed309bfe88c279ea593024c46af51219a9 Mon Sep 17 00:00:00 2001 From: Gert Mertes Date: Mon, 12 Aug 2024 12:20:43 +0000 Subject: [PATCH 1/3] fix: mlflow auth status log --- src/anemoi/training/diagnostics/mlflow/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/anemoi/training/diagnostics/mlflow/logger.py b/src/anemoi/training/diagnostics/mlflow/logger.py index 4c9bf7ff..4fac38e6 100644 --- a/src/anemoi/training/diagnostics/mlflow/logger.py +++ b/src/anemoi/training/diagnostics/mlflow/logger.py @@ -374,7 +374,7 @@ def __init__( if offline: LOGGER.info("MLflow is logging offline.") else: - LOGGER.info("MLflow token authentication {'enabled' if enabled else 'disabled'} for %s", tracking_uri) + LOGGER.info("MLflow token authentication %s for %s", "enabled" if enabled else "disabled", tracking_uri) self.auth.authenticate() health_check(tracking_uri) From 7113de97e595fc6d0d9c743c830953dad7affc14 Mon Sep 17 00:00:00 2001 From: "S. Hahner" <44293258+sahahner@users.noreply.github.com> Date: Tue, 13 Aug 2024 14:13:51 +0200 Subject: [PATCH 2/3] Fix minor errors in two callback plot functions (#24) * fix: minor errors in two callback plot functions * changelog --- CHANGELOG.md | 1 + src/anemoi/training/diagnostics/callbacks/__init__.py | 2 +- src/anemoi/training/diagnostics/plots.py | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ec1511d..17df3a46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ Keep it human-readable, your future self will thank you! - `__future__` annotations for typehints - Added Typehints where missing - Added Changelog + - Correct errors in callback plots ### Changed diff --git a/src/anemoi/training/diagnostics/callbacks/__init__.py b/src/anemoi/training/diagnostics/callbacks/__init__.py index 15b9d554..2153b7da 100644 --- a/src/anemoi/training/diagnostics/callbacks/__init__.py +++ b/src/anemoi/training/diagnostics/callbacks/__init__.py @@ -488,7 +488,7 @@ def on_validation_batch_end( batch_idx: int, ) -> None: if batch_idx % self.plot_frequency == 0: - self.plot(trainer, pl_module, outputs, batch, epoch=trainer.current_epoch) + self.plot(trainer, pl_module, outputs, batch, batch_idx, epoch=trainer.current_epoch) class PlotSample(BasePlotCallback): diff --git a/src/anemoi/training/diagnostics/plots.py b/src/anemoi/training/diagnostics/plots.py index a0922fe5..a3aec61f 100644 --- a/src/anemoi/training/diagnostics/plots.py +++ b/src/anemoi/training/diagnostics/plots.py @@ -291,8 +291,9 @@ def plot_histogram( # Visualization trick for tp if variable_name in {"tp", "cp"}: - hist_yt *= bins_yt[:-1] - hist_yp *= bins_yp[:-1] + # in-place multiplication does not work here because variables are different numpy types + hist_yt = hist_yt * bins_yt[:-1] + hist_yp = hist_yp * bins_yp[:-1] # Plot the modified histogram ax[plot_idx].bar(bins_yt[:-1], hist_yt, width=np.diff(bins_yt), color="blue", alpha=0.7, label="Truth (ERA5)") ax[plot_idx].bar(bins_yp[:-1], hist_yp, width=np.diff(bins_yp), color="red", alpha=0.7, label="Anemoi") From 6d289fdb9f8e63a4c5d32f2958aa3fce211f2b61 Mon Sep 17 00:00:00 2001 From: Gert Mertes <13658335+gmertes@users.noreply.github.com> Date: Wed, 14 Aug 2024 13:53:12 +0100 Subject: [PATCH 3/3] Fix hyperparameter logging crash with missing config values (#19) * fix: logging hyperparameters with missing config values * chore: update changelog --- CHANGELOG.md | 1 + src/anemoi/training/diagnostics/mlflow/logger.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17df3a46..a494f12f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,7 @@ Keep it human-readable, your future self will thank you! - Moved callbacks into folder to fascilitate future refactor - Adjusted PyPI release infrastructure to common ECMWF workflow - Bumped versions in Pre-commit hooks + - Fix crash when logging hyperparameters with missing values in the config ### Removed - Dependency on mlflow-export-import diff --git a/src/anemoi/training/diagnostics/mlflow/logger.py b/src/anemoi/training/diagnostics/mlflow/logger.py index 4fac38e6..7854c172 100644 --- a/src/anemoi/training/diagnostics/mlflow/logger.py +++ b/src/anemoi/training/diagnostics/mlflow/logger.py @@ -27,6 +27,7 @@ from pytorch_lightning.utilities.rank_zero import rank_zero_only from anemoi.training.diagnostics.mlflow.auth import TokenAuth +from anemoi.training.utils.jsonify import map_config_to_primitives if TYPE_CHECKING: from argparse import Namespace @@ -454,6 +455,11 @@ def log_hyperparams(self, params: dict[str, Any] | Namespace) -> None: """Overwrite the log_hyperparams method to flatten config params using '.'.""" if self._flag_log_hparams: params = _convert_params(params) + + # this is needed to resolve optional missing config values to a string, instead of raising a missing error + if config := params.get("config"): + params["config"] = map_config_to_primitives(config) + params = _flatten_dict(params, delimiter=".") # Flatten dict with '.' to not break API queries params = self._clean_params(params)