Charts

MAIA/dashboard_utils.py

@@ -1180,7 +1180,8 @@ def register_user_in_keycloak(email, settings):
                                             'temporary':True,
                                             'value': temp_password}],                         
                                               })
-    send_approved_registration_email(email, maia_login_url, temp_password)
+    if "email_account" in os.environ and "email_password" in os.environ and "email_smtp_server" in os.environ:
+        send_approved_registration_email(email, maia_login_url, temp_password)
 
 def register_group_in_keycloak(group_id, settings):
     """

MAIA/maia_admin.py

@@ -758,3 +758,171 @@ def create_minio_operator_values(config_folder, project_id, cluster_config_dict)
         "version": minio_operator_values["chart_version"],
         "values": str(Path(config_folder).joinpath(project_id, "minio_operator_values", "minio_operator_values.yaml"))
     }
+
+def create_maia_dashboard_values(config_folder, project_id, cluster_config_dict, maia_config_dict):
+    """
+    Create MAIA dashboard values for Helm chart deployment.
+
+    Parameters
+    ----------
+    config_folder : str
+        The path to the configuration folder.
+    project_id : str
+        The project identifier.
+    cluster_config_dict : dict
+        Dictionary containing cluster configuration details.
+    maia_config_dict : dict
+        Dictionary containing MAIA configuration details.
+
+    Returns
+    -------
+    dict
+        A dictionary containing the namespace, release name, chart name, repository URL, chart version, 
+        and the path to the generated values YAML file.
+    """
+
+    maia_dashboard_values = {
+        "namespace": "maia-dashboard",
+        "repo_url": "https://kthcloud.github.io/MAIA/",
+        "chart_name": "maia-dashboard",
+        "chart_version": "0.1.2",
+    }
+
+
+
+
+    maia_dashboard_values.update({
+        "image": {
+            "repository": "registry." + cluster_config_dict["domain"] + "/maia/maia-dashboard",
+            "pullPolicy": "IfNotPresent",
+            "tag": "1.0"
+        },
+        "imagePullSecrets": [
+            {
+                "name": "registry."+ cluster_config_dict["domain"]
+            }
+        ],
+        "storageClass": cluster_config_dict["storage_class"],
+        "ingress": {
+            "enabled": True,
+            "className": cluster_config_dict["ingress_class"],
+            "annotations": {
+            },
+            "hosts": [
+                { "host": cluster_config_dict["domain"],
+                    "paths": [
+                        { "path": "/maia/",
+                            "pathType": "Prefix"
+                        }
+                    ]
+                    }
+            ],
+            "tls": [
+                { "hosts": [
+                    cluster_config_dict["domain"]
+                    ]
+                }
+            ]
+        },
+    "gpuList": maia_config_dict["gpu_list"],
+    "dashboard": {
+        "host": cluster_config_dict["domain"],
+        "keycloak": {
+            "client_id": "maia",
+            "client_secret": cluster_config_dict["keycloak_maia_client_secret"],
+            "url": "https://iam." + cluster_config_dict["domain"]+"/",
+            "realm": "maia",
+            "username": "admin"
+        },
+        "argocd_server": "https://argocd." + cluster_config_dict["domain"],
+        "argocd_cluster_name": cluster_config_dict["cluster_name"],
+        "local_db_path": "/etc/MAIA-Dashboard/db"
+    },
+    "argocd_namespace": maia_config_dict["argocd_namespace"],
+    "admin_group_ID": maia_config_dict["admin_group_ID"],
+    "core_project_chart": "maia-core-project",
+    "core_project_repo": "https://kthcloud.github.io/MAIA/",
+    "core_project_version": "1.0.0",
+    "admin_project_chart": "maia-admin-project",
+    "admin_project_repo": "https://kthcloud.github.io/MAIA/",
+    "admin_project_version": "1.0.0",
+    "maia_project_chart": "maia-project",
+    "maia_project_repo": "https://kthcloud.github.io/MAIA/",
+    "maia_project_version": "1.0.0",
+    "maia_workspace_version": "1.6c",
+    "maia_workspace_image": "kthcloud/maia-workspace-notebook-ssh-addons",
+    "maia_monai_toolkit_image": "kthcloud/maia-monai-toolkit:3.0.1",    
+    "name": "maia-dashboard",
+    "dockerRegistrySecretName": "registry."+ cluster_config_dict["domain"],
+    "dockerRegistryUsername": cluster_config_dict["docker_username"],
+    "dockerRegistryPassword": cluster_config_dict["docker_password"],
+    "dockerRegistryEmail": cluster_config_dict["docker_email"],
+    "dockerRegistryServer": "registry."+ cluster_config_dict["domain"],
+    }
+    )
+
+    maia_dashboard_values["clusters"] = [
+        cluster_config_dict
+    ]
+
+    if "discord_url" in maia_config_dict:
+        maia_dashboard_values["dashboard"]["discord_url"] = maia_config_dict["discord_url"]
+    if "discord_signup_url" in maia_config_dict:
+        maia_dashboard_values["dashboard"]["discord_signup_url"] = maia_config_dict["discord_signup_url"]
+
+    debug = True
+
+    if debug:
+
+        maia_dashboard_values["env"] = [
+            { "name": "DEBUG", "value": "True" },
+            { "name": "CLUSTER_CONFIG_PATH", "value": "/etc/MAIA-Dashboard/config" },
+            { "name": "CONFIG_PATH", "value": "/etc/MAIA-Dashboard/config" },
+            { "name": "MAIA_CONFIG_PATH", "value": "/etc/MAIA-Dashboard/config/maia_config.yaml" }
+        ]
+        maia_dashboard_values["dashboard"]["local_config_path"] = "/etc/MAIA-Dashboard/config"
+    else:
+        db_password = token_urlsafe(16)
+        maia_dashboard_values["dashboard"]["local_config_path"] = "/mnt/dashboard-config"
+        maia_dashboard_values["env"] = [
+            { "name": "DEBUG", "value": "False" },
+            { "name": "CLUSTER_CONFIG_PATH", "value": "/mnt/dashboard-config" },
+            { "name": "CONFIG_PATH", "value": "/mnt/dashboard-config" },
+            { "name": "MAIA_CONFIG_PATH", "value": "/mnt/dashboard-config/maia_config.yaml" },
+            { "name": "DB_ENGINE", "value": "mysql" },
+            { "name": "DB_NAME", "value": "mysql" },
+            { "name": "DB_HOST", "value": "maia-admin-maia-dashboard-mysql" },
+            { "name": "DB_PORT", "value": "3306" },
+            { "name": "DB_USERNAME", "value": "maia-admin" },
+            { "name": "DB_PASS", "value": db_password },
+        ]
+        maia_dashboard_values["mysql"] = {
+            "enabled": True,
+            "mysqlRootPassword": db_password,
+            "mysqlUser": "maia-admin",
+            "mysqlPassword": db_password,
+            "mysqlDatabase": "mysql"
+        }
+
+    if maia_config_dict["email_server_credentials"]:
+        maia_dashboard_values["env"].extend([
+            { "name": "email_account", "value": maia_config_dict["email_server_credentials"]["email_account"] },
+            { "name": "email_password", "value": maia_config_dict["email_server_credentials"]["email_password"] },
+            { "name": "email_smtp_server", "value": maia_config_dict["email_server_credentials"]["email_smtp_server"] }
+        ])
+
+    Path(config_folder).joinpath(project_id, "maia_dashboard_values").mkdir(parents=True, exist_ok=True)
+    with open(Path(config_folder).joinpath(project_id, "maia_dashboard_values", "maia_dashboard_values.yaml"), "w") as f:
+        f.write(OmegaConf.to_yaml(maia_dashboard_values))
+
+    return {
+        "namespace": maia_dashboard_values["namespace"],
+        "release": f"{project_id}-dashboard",
+        "chart": maia_dashboard_values["chart_name"],
+        "repo": maia_dashboard_values["repo_url"],
+        "version": maia_dashboard_values["chart_version"],
+        "values": str(Path(config_folder).joinpath(project_id, "maia_dashboard_values", "maia_dashboard_values.yaml"))
+    }
+
+
+

MAIA/maia_docker_images.py

@@ -0,0 +1,83 @@
+from pathlib import Path
+import yaml
+
+def deploy_maia_kaniko(namespace, config_folder, cluster_config_dict, release_name, project_id, registry_url, registry_secret_name, image_name, image_tag, subpath, build_args=None):
+    """
+    Deploys a Kaniko job for building and pushing Docker images to a specified registry.
+
+    Parameters
+    ----------
+    namespace : str
+        The Kubernetes namespace where the Kaniko job will be deployed.
+    config_folder : str
+        The folder path where the configuration files will be stored.
+    cluster_config_dict : dict
+        Dictionary containing cluster configuration details, including storage class.
+    release_name : str
+        The release name for the Kaniko job.
+    project_id : str
+        The project identifier.
+    registry_url : str
+        The URL of the Docker registry where the image will be pushed.
+    registry_secret_name : str
+        The name of the Kubernetes secret for accessing the Docker registry.
+    image_name : str
+        The name of the Docker image to be built.
+    image_tag : str
+        The tag of the Docker image to be built.
+    subpath : str
+        The subpath of the repository where the Dockerfile is located.
+    build_args : list, optional
+        A list of build arguments to be passed to the Kaniko job.
+
+    Returns
+    -------
+    dict
+        A dictionary containing deployment details including namespace, release name, chart name, repo URL, chart version, and values file path.
+    """
+
+    kaniko_values = {
+        "chart_name": "mkg-kaniko",
+        "repo_url": "https://kthcloud.github.io/MAIA/",
+        "chart_version": "1.0.3",
+        "namespace": "mkg-kaniko",
+    }
+
+
+    kaniko_values.update(
+        {
+            "docker_registry_secret": registry_secret_name,
+            "pvc": {
+                "pvc_type": cluster_config_dict["storage_class"],
+                "size": "10Gi",
+            },
+            "args": [
+                "--dockerfile=Dockerfile",
+                "--context=git://github.com/kthcloud/MAIA.git", #  #refs/heads/mybranch
+                "--context-sub-path="+subpath,
+                "--destination={}/{}:{}".format(registry_url, image_name, image_tag),
+                "--cache=true",
+                "--cache-dir=/cache",
+            ],
+        }
+    )
+
+    if build_args:
+        for build_arg in build_args:
+            kaniko_values["args"].append(f"--build-arg={build_arg}")
+
+
+    release_name_values = release_name.replace("-", "_")
+    Path(config_folder).joinpath(project_id, f"{release_name_values}_values").mkdir(parents=True, exist_ok=True)
+    with open(Path(config_folder).joinpath(project_id, f"{release_name_values}_values", f"{release_name_values}_values.yaml"), "w") as f:
+        yaml.dump(kaniko_values, f)
+
+
+    return {
+        "namespace": namespace,
+        "release":  f"{project_id}-{release_name}",
+        "chart": kaniko_values["chart_name"],
+        "repo": kaniko_values["repo_url"],
+        "version": kaniko_values["chart_version"],
+        "values": str(Path(config_folder).joinpath(project_id, f"{release_name_values}_values", f"{release_name_values}_values.yaml"))
+    }

MAIA_scripts/MAIA_initialize_config.py

@@ -110,6 +110,12 @@ def create_configuration(cluster_config, config_folder):
         "enabled": True,
         "token": ""
     }
+
+    cluster_config_dict["imagePullSecrets"] = "registry."+cluster_config_dict["domain"]
+    cluster_config_dict["docker_server"] = "registry."+cluster_config_dict["domain"]
+    cluster_config_dict["docker_username"] = "CHANGEME_ROBOT"
+    cluster_config_dict["docker_password"] = "CHANGEME_ROBOT_PASSWORD"
+    cluster_config_dict["docker_email"] = "CHANGEME_ROBOT_EMAIL"
 
 
     with open(os.path.join(config_folder,"MAIA_realm_template.json"), "r") as f:

MAIA_scripts/MAIA_install_admin_toolkit.py

@@ -3,7 +3,7 @@
 import click
 import yaml
 from omegaconf import OmegaConf
-from MAIA.maia_admin import install_maia_project, create_harbor_values, create_keycloak_values, create_loginapp_values, create_minio_operator_values, create_maia_admin_toolkit_values
+from MAIA.maia_admin import install_maia_project, create_harbor_values, create_keycloak_values, create_loginapp_values, create_minio_operator_values, create_maia_admin_toolkit_values, create_maia_dashboard_values
 from hydra import initialize, initialize_config_dir
 from hydra import compose as hydra_compose
 import datetime
@@ -115,6 +115,7 @@ def install_maia_admin_toolkit(maia_config_file, cluster_config, config_folder):
     helm_commands.append(create_loginapp_values(config_folder, project_id,cluster_config_dict))
     helm_commands.append(create_minio_operator_values(config_folder, project_id,cluster_config_dict))
     helm_commands.append(create_maia_admin_toolkit_values(config_folder, project_id,cluster_config_dict, maia_config_dict=maia_config_dict))
+    helm_commands.append(create_maia_dashboard_values(config_folder, project_id,cluster_config_dict, maia_config_dict=maia_config_dict))
 
 
     for helm_command in helm_commands:
@@ -144,6 +145,7 @@ def install_maia_admin_toolkit(maia_config_file, cluster_config, config_folder):
                 {"loginapp_values": "loginapp_values"},
                 {"minio_operator_values": "minio_operator_values"},
                 {"maia_admin_toolkit_values": "maia_admin_toolkit_values"},
+                {"maia_dashboard_values": "maia_dashboard_values"}
 
          ],
         "argo_namespace": maia_config_dict["argocd_namespace"],
@@ -181,13 +183,17 @@ def install_maia_admin_toolkit(maia_config_file, cluster_config, config_folder):
 
 
         project_chart = maia_config_dict["admin_project_chart"]
-        asyncio.run(install_maia_project(project_id, Path(config_folder).joinpath(project_id, f"{project_id}_values.yaml"),maia_config_dict["argocd_namespace"], project_chart))
+        project_repo = maia_config_dict["admin_project_repo"]
+        project_version = maia_config_dict["admin_project_version"]
+        asyncio.run(install_maia_project(project_id, Path(config_folder).joinpath(project_id, f"{project_id}_values.yaml"),maia_config_dict["argocd_namespace"], project_chart, project_repo=project_repo, project_version=project_version))
     else:
         print("Upgrading MAIA Admin Toolkit")
 
 
         project_chart = maia_config_dict["admin_project_chart"]
-        asyncio.run(install_maia_project(project_id, Path(config_folder).joinpath(project_id, f"{project_id}_values.yaml"),maia_config_dict["argocd_namespace"], project_chart))
+        project_repo = maia_config_dict["admin_project_repo"]
+        project_version = maia_config_dict["admin_project_version"]
+        asyncio.run(install_maia_project(project_id, Path(config_folder).joinpath(project_id, f"{project_id}_values.yaml"),maia_config_dict["argocd_namespace"], project_chart, project_repo=project_repo, project_version=project_version))
 
 
 if __name__ == "__main__":

MAIA_scripts/MAIA_install_core_toolkit.py

@@ -193,13 +193,17 @@ def install_maia_core_toolkit(maia_config_file, cluster_config, config_folder):
 
 
         project_chart = maia_config_dict["core_project_chart"]
-        asyncio.run(install_maia_project(project_id, Path(config_folder).joinpath(project_id, f"{project_id}_values.yaml"),maia_config_dict["argocd_namespace"], project_chart))
+        project_repo = maia_config_dict["core_project_repo"]
+        project_version = maia_config_dict["core_project_version"]
+        asyncio.run(install_maia_project(project_id, Path(config_folder).joinpath(project_id, f"{project_id}_values.yaml"),maia_config_dict["argocd_namespace"], project_chart, project_repo=project_repo, project_version=project_version))
     else:
         print("Upgrading MAIA Core Toolkit")
 
 
         project_chart = maia_config_dict["core_project_chart"]
-        asyncio.run(install_maia_project(project_id, Path(config_folder).joinpath(project_id, f"{project_id}_values.yaml"),maia_config_dict["argocd_namespace"], project_chart))
+        project_repo = maia_config_dict["core_project_repo"]
+        project_version = maia_config_dict["core_project_version"]
+        asyncio.run(install_maia_project(project_id, Path(config_folder).joinpath(project_id, f"{project_id}_values.yaml"),maia_config_dict["argocd_namespace"], project_chart, project_repo=project_repo, project_version=project_version))
 
 
 if __name__ == "__main__":

docker/MAIA-Workspace/Tutorials/DL_tools/Image segmentation with MONAI/README.md

@@ -6,8 +6,8 @@ This tutorial/lab is designed to introduce you to the concepts of medical image
 ## Tutorial structure
 The tutorial is divided into three parts:
 
-1. **Part 1: Introduction to Segmentation and U-Net:** Segmentation and UNET.ipynb
-2. **Part 2: Visualizing attention maps:** Attention Maps.ipynb
-3. **Part 3: Training a U-Net model with MONAI:** MONAI.ipynb
+1. **Part 1: Introduction to Segmentation and U-Net:** [Segmentation and UNET](Segmentation%20and%20UNET.ipynb)
+2. **Part 2: Visualizing attention maps:** [Attention Maps](Attention%20maps.ipynb)
+3. **Part 3: Training a U-Net model with MONAI:** [MONAI](MONAI.ipynb)
 
 The lab requires you to code some part in python files, you can use the VSCode IDE on JupyterHub to do so more conveniently.

docker/MAIA-Workspace/Tutorials/DL_tools/PyTorch Convolutional Networks/README.md

@@ -10,9 +10,9 @@ You will be using the [OrganSMNIST](https://github.com/maia-ai/cm2003/tree/main/
 
 
 ## Getting started
-1. Start with the Convolutional Architectures notebook. This notebook will guide you through the process of training a convolutional neural network on the OrganSMNIST dataset.
-2. Continue with the Improving training and transfer learning notebook. This notebook will guide you through the process of improving the performance of your model using regularization techniques, weight decay, data augmentation, and transfer learning.
-3. Finally, explore the different models and techniques we have provided you with. You can use the Model playground notebook to test out different models and techniques.
+1. Start with the Convolutional Architectures notebook. This notebook will guide you through the process of training a convolutional neural network on the OrganSMNIST dataset. [Convolutional Architectures](Convolutional%20architectures.ipynb)
+2. Continue with the Improving training and transfer learning notebook. This notebook will guide you through the process of improving the performance of your model using regularization techniques, weight decay, data augmentation, and transfer learning. [Improving training and transfer learning](Improving%20training%20and%20transfer%20learning.ipynb)
+3. Finally, explore the different models and techniques we have provided you with. You can use the Model playground notebook to test out different models and techniques. [Model playground](Model%20playground.ipynb)
 
 ## Submission