Model1024_HPT_EXP_ATT4_CosineAnnealingWarmRestarts_40_epochs.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kvvmWRaJllNP"
      },
      "source": [
        "Available: 47.68 compute units"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sY48Ap7wK2Sn"
      },
      "source": [
        "PNEUMOTHORAX SEGMENTATION USING SIIM-ACR DATASET"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZMmEp3gqUyn1",
        "outputId": "67a53414-3c93-41e9-f478-f55460c7c064"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y_qWc-LUIJCv",
        "outputId": "25d6cc91-6060-415b-8626-2aadb269d8df"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting git+https://github.com/qubvel/segmentation_models.pytorch\n",
            "  Cloning https://github.com/qubvel/segmentation_models.pytorch to /tmp/pip-req-build-9b80v2ot\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/qubvel/segmentation_models.pytorch /tmp/pip-req-build-9b80v2ot\n",
            "  Resolved https://github.com/qubvel/segmentation_models.pytorch to commit bd7f877fec51309a1a7de7670aed232954450037\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: torchvision>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from segmentation_models_pytorch==0.3.4.dev0) (0.18.0+cu121)\n",
            "Collecting pretrainedmodels==0.7.4 (from segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Downloading pretrainedmodels-0.7.4.tar.gz (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.8/58.8 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting efficientnet-pytorch==0.7.1 (from segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Downloading efficientnet_pytorch-0.7.1.tar.gz (21 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting timm==0.9.7 (from segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Downloading timm-0.9.7-py3-none-any.whl (2.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m25.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from segmentation_models_pytorch==0.3.4.dev0) (4.66.4)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from segmentation_models_pytorch==0.3.4.dev0) (9.4.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from segmentation_models_pytorch==0.3.4.dev0) (1.16.0)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0) (2.3.0+cu121)\n",
            "Collecting munch (from pretrainedmodels==0.7.4->segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Downloading munch-4.0.0-py2.py3-none-any.whl (9.9 kB)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from timm==0.9.7->segmentation_models_pytorch==0.3.4.dev0) (6.0.1)\n",
            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from timm==0.9.7->segmentation_models_pytorch==0.3.4.dev0) (0.23.4)\n",
            "Requirement already satisfied: safetensors in /usr/local/lib/python3.10/dist-packages (from timm==0.9.7->segmentation_models_pytorch==0.3.4.dev0) (0.4.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torchvision>=0.5.0->segmentation_models_pytorch==0.3.4.dev0) (1.25.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0) (3.15.4)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0) (4.12.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0) (1.12.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0) (2023.6.0)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "Collecting nvidia-curand-cu12==10.3.2.106 (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "Collecting nvidia-nccl-cu12==2.20.5 (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n",
            "Collecting nvidia-nvtx-cu12==12.1.105 (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "Requirement already satisfied: triton==2.3.0 in /usr/local/lib/python3.10/dist-packages (from torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0) (2.3.0)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.5.82-py3-none-manylinux2014_x86_64.whl (21.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m44.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->timm==0.9.7->segmentation_models_pytorch==0.3.4.dev0) (24.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->timm==0.9.7->segmentation_models_pytorch==0.3.4.dev0) (2.31.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0) (2.1.5)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->timm==0.9.7->segmentation_models_pytorch==0.3.4.dev0) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->timm==0.9.7->segmentation_models_pytorch==0.3.4.dev0) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->timm==0.9.7->segmentation_models_pytorch==0.3.4.dev0) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->timm==0.9.7->segmentation_models_pytorch==0.3.4.dev0) (2024.6.2)\n",
            "Requirement already satisfied: mpmath<1.4.0,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->efficientnet-pytorch==0.7.1->segmentation_models_pytorch==0.3.4.dev0) (1.3.0)\n",
            "Building wheels for collected packages: segmentation_models_pytorch, efficientnet-pytorch, pretrainedmodels\n",
            "  Building wheel for segmentation_models_pytorch (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for segmentation_models_pytorch: filename=segmentation_models_pytorch-0.3.4.dev0-py3-none-any.whl size=109566 sha256=d6eb0d24b842831c3a7c17ca1299d7168b197a6c8ffa17aaf7ea46f73fdf27b3\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-ow7jgreg/wheels/1a/49/5f/858bc2741660e381e83f1d8b297edc4d9f0561f29becaee577\n",
            "  Building wheel for efficientnet-pytorch (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for efficientnet-pytorch: filename=efficientnet_pytorch-0.7.1-py3-none-any.whl size=16429 sha256=6ffc6936a60e260ca264cf1c0e61d10809e0e7fcb637365e63af98c9fbb6117a\n",
            "  Stored in directory: /root/.cache/pip/wheels/03/3f/e9/911b1bc46869644912bda90a56bcf7b960f20b5187feea3baf\n",
            "  Building wheel for pretrainedmodels (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pretrainedmodels: filename=pretrainedmodels-0.7.4-py3-none-any.whl size=60945 sha256=55c60008eafbb234d1a0f90dcde3a9683c250937df931c917f9e6c74f2fd92d6\n",
            "  Stored in directory: /root/.cache/pip/wheels/35/cb/a5/8f534c60142835bfc889f9a482e4a67e0b817032d9c6883b64\n",
            "Successfully built segmentation_models_pytorch efficientnet-pytorch pretrainedmodels\n",
            "Installing collected packages: nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, munch, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, efficientnet-pytorch, timm, pretrainedmodels, segmentation_models_pytorch\n",
            "Successfully installed efficientnet-pytorch-0.7.1 munch-4.0.0 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.5.82 nvidia-nvtx-cu12-12.1.105 pretrainedmodels-0.7.4 segmentation_models_pytorch-0.3.4.dev0 timm-0.9.7\n"
          ]
        }
      ],
      "source": [
        "!pip install git+https://github.com/qubvel/segmentation_models.pytorch"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y7HFiESzDWcY"
      },
      "source": [
        "## Importing Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "CUKZcMbBK2Ss"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import torch\n",
        "import csv\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from PIL import Image\n",
        "import cv2\n",
        "import json\n",
        "from pathlib import Path\n",
        "import matplotlib.pyplot as plt\n",
        "# import torchvision\n",
        "from torch.utils.data import DataLoader\n",
        "import torch.nn as nn\n",
        "from collections import defaultdict\n",
        "import albumentations as albu\n",
        "from albumentations.pytorch.transforms import ToTensorV2\n",
        "import torch.nn.functional as F\n",
        "from torch import nn\n",
        "from torch.autograd import Variable\n",
        "\n",
        "import torch.nn.functional as F\n",
        "import torchvision.transforms.functional as TF\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from torch.utils.data.sampler import Sampler\n",
        "from torch.nn.utils import clip_grad_norm_\n",
        "from tqdm import tqdm\n",
        "from sklearn.metrics import confusion_matrix, accuracy_score, recall_score, f1_score\n",
        "import torchvision.models as models # NEW MARCH-JUNE EDIT\n",
        "\n",
        "try:\n",
        "    from itertools import ifilterfalse\n",
        "except ImportError:  # py3k\n",
        "    from itertools import filterfalse"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xfcsl-YW_ck9"
      },
      "source": [
        "## Config - 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "-vx6RpZ8_bba"
      },
      "outputs": [],
      "source": [
        "IMG_SIZE         = 1024\n",
        "DIR              = \"/content/drive/MyDrive/datasets\"\n",
        "DATA_DIR         = Path(DIR)\n",
        "TRAIN_IMG_DIR    = Path(\"/content/drive/MyDrive/datasets/train_png\")\n",
        "TRAIN_LBL_DIR    = Path(\"/content/drive/MyDrive/datasets/mask\")\n",
        "DATA_FRAME_PATH  = \"/content/drive/MyDrive/datasets/RLE_kfold.csv\"\n",
        "KFOLD_PATH       = \"\"\n",
        "TRAIN_BATCH_SIZE = 8\n",
        "VALID_BATCH_SIZE = 8\n",
        "BATCH_SIZE       = 8\n",
        "EPOCHS           = 50\n",
        "# Path for pretrained model weights\n",
        "TRAINING_MODEL_PATH = \"/content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/m_1024_CHECKPOINTS_epoch15_bst_model1024_fold4_0.8294.tar\"\n",
        "USE_SAMPLER      = True\n",
        "POSTIVE_PERC     = 0.4\n",
        "DEVICE           = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "PRETRAINED       = True # False means we're using ImageNet weights, so essentially, it's pretrained & never from scratch!!!!!\n",
        "LEARNING_RATE    = 0.00001\n",
        "NUM_WORKERS      = 12\n",
        "USE_CRIT         = True\n",
        "FOLD_ID          = 4\n",
        "EVAL_METRICS      = [\"metric - it calculates dice coefficient.\"]\n",
        "\n",
        "# Regularization Settings\n",
        "EARLY_STOPPING_PATIENCE = 12\n",
        "L2_WEIGHT_DECAY  = 0.000005\n",
        "GRADIENT_CLIPPING = True\n",
        "GRADIENT_CLIPPING_THRESHOLD = 0.1 # for model trained on 1024x1024\n",
        "\n",
        "# IF U DON'T WANT TO CHANGE Learning rate from previous experiment then set this to True.\n",
        "OPTIMIZER_LOAD = True\n",
        "# U MUST VERIFY IF LR IS GETTING SET PROPERLY FOR THE SCHEDULER IN THE \"OPTIMIZER & SCHEDULER\" SECTION OF THE NOTEBOOK.\n",
        "\n",
        "# Learning Rate Scheduler Settings\n",
        "SCHEDULER        = \"CosineAnnealingWarmRestarts\"\n",
        "if SCHEDULER == \"ReduceLROnPlateau\":\n",
        "    SCHEDULER_PARAMS = {'factor': 0.1, 'patience': 2, 'threshold': 0.0000001, 'min_lr': 0.0000001}\n",
        "elif SCHEDULER == \"CosineAnnealingWarmRestarts\":\n",
        "    SCHEDULER_PARAMS = {'T_0': 1, 'T_mult': 2, 'eta_min': 0.0000001}\n",
        "elif SCHEDULER == \"CosineAnnealingLR\":\n",
        "    SCHEDULER_PARAMS = {'T_max': 8, 'eta_min': 0.0000001}\n",
        "\n",
        "# Thresholds for 1024x1024 is there along with div by 2 values and div by 4 values\n",
        "TRIPLET_THRESHOLDS = [  [0.6, 500.0, 0.35], [0.67, 500.0, 0.37], [0.75, 500.0, 0.3],\n",
        "                        [0.75, 500.0, 0.4], [0.75, 1000.0, 0.3], [0.75, 1000.0, 0.4],\n",
        "                        [0.6, 1000.0, 0.3], [0.6, 1000.0, 0.4], [0.6, 1500.0, 0.3],\n",
        "                        [0.6, 1500.0, 0.4], [0.6, 250.0, 0.35], [0.67, 250.0, 0.37],\n",
        "                        [0.75, 250.0, 0.3], [0.75, 250.0, 0.4], [0.75, 500.0, 0.3],\n",
        "                        [0.75, 500.0, 0.4], [0.6, 500.0, 0.3], [0.6, 500.0, 0.4],\n",
        "                        [0.6, 750.0, 0.3], [0.6, 750.0, 0.4], [0.6, 1000, 0.35],\n",
        "                        [0.67, 1000, 0.37], [0.75, 1000, 0.3], [0.75, 1000, 0.4],\n",
        "                        [0.75, 2000, 0.3], [0.75, 2000, 0.4], [0.6, 2000, 0.3],\n",
        "                        [0.6, 2000, 0.4], [0.6, 3000, 0.3], [0.6, 3000, 0.4]       ]\n",
        "\n",
        "\n",
        "\"\"\"\n",
        "    \"-------------------------------SAVING losses & metrics-------------------------------------\"\n",
        "\"\"\"\n",
        "ACCOUNT         = \"Neeshanth\" # \"Neeshanth\" or \"wonder_boys\" or others\n",
        "PURPOSE         = \"Model1024_HPT_attempting_better_convergence\" # HPT_Exploration/HPT_Finetuning/training/inference/hyperparameter-tuning or hpt OR ANYTHING MORE SPECIFIC\n",
        "EXP_NO          = 'ATT_4.1_BS_8' # hpt experiment with changed settings\n",
        "PHASE           = \"3_step_1_explore\"\n",
        "\n",
        "EFFECTIVE_BATCH_SIZE = 8 # accumulation_steps * BATCH_SIZE\n",
        "\n",
        "\n",
        "if ACCOUNT == \"Neeshanth\":\n",
        "    # Save Config.txt file\n",
        "    CONFIG_FILE_LOC = f\"/content/drive/MyDrive/Saved Models/model1024/{PURPOSE}_EXP_{EXP_NO}_PHASE_{PHASE}/Config.txt\"\n",
        "\n",
        "    # Save model checkpoint using early stopping class\n",
        "    model_checkpoint_path = f\"/content/drive/MyDrive/Saved Models/model1024/{PURPOSE}_EXP_{EXP_NO}_PHASE_{PHASE}/m_{IMG_SIZE}_CHECKPOINTS\"\n",
        "\n",
        "    # Save batch wise comboloss during training\n",
        "    store_batch_training_details_path = f\"/content/drive/MyDrive/Saved Models/model1024/{PURPOSE}_EXP_{EXP_NO}_PHASE_{PHASE}/{PURPOSE}_train_m{IMG_SIZE}_b{EFFECTIVE_BATCH_SIZE}\"\n",
        "    name_of_batch_training_details_csv = f\"{PURPOSE}_TRAIN_{IMG_SIZE}_b_{EFFECTIVE_BATCH_SIZE}_epoch_\" # epoch no. will be added to the end\n",
        "\n",
        "    # Save batch wise comboloss during validation\n",
        "    store_batch_validation_details_path = f\"/content/drive/MyDrive/Saved Models/model1024/{PURPOSE}_EXP_{EXP_NO}_PHASE_{PHASE}/{PURPOSE}_vali_m{IMG_SIZE}_b{EFFECTIVE_BATCH_SIZE}\"\n",
        "    name_of_batch_validation_details_csv = f\"{PURPOSE}_{IMG_SIZE}_b_{EFFECTIVE_BATCH_SIZE}_Validation_Metrics\"\n",
        "\n",
        "    # Save epoch wise train and val losses to monitor overfitting\n",
        "    save_progress_path = f\"/content/drive/MyDrive/Saved Models/model1024/{PURPOSE}_EXP_{EXP_NO}_PHASE_{PHASE}/{PURPOSE}_progress.csv\"\n",
        "\n",
        "    # Save epoch wise dice coefficient\n",
        "    save_dice_score_path = f\"/content/drive/MyDrive/Saved Models/model1024/{PURPOSE}_EXP_{EXP_NO}_PHASE_{PHASE}/{PURPOSE}_dice_scores.csv\"\n",
        "\n",
        "    # Save bce, dice & focal losses separately after training is done\n",
        "    save_3losses_path = f\"/content/drive/MyDrive/Saved Models/model1024/{PURPOSE}_EXP_{EXP_NO}_PHASE_{PHASE}/{PURPOSE}_all_loss_values.csv\"\n",
        "\n",
        "    # Save best thresholds & their dice scores - done according to Triplet Scheme of Binarization\n",
        "    save_best_thresholds_path = f\"/content/drive/MyDrive/Saved Models/model1024/{PURPOSE}_EXP_{EXP_NO}_PHASE_{PHASE}/{PURPOSE}_best_checkpoint_thresholds.csv\"\n",
        "\n",
        "if ACCOUNT == \"wonder_boys\":\n",
        "    # Save batch wise comboloss during training\n",
        "    store_batch_training_details_path = \"/content/drive/MyDrive/Saved Models/Model_{EXP_NO}/hpt_train_m512_b32\"\n",
        "    name_of_batch_training_details_csv = f\"{PURPOSE}_TRAIN_{IMG_SIZE}_b_{EFFECTIVE_BATCH_SIZE}_epoch_\" # epoch no. will be added to the end\n",
        "\n",
        "    # Save batch wise comboloss during validation\n",
        "    store_batch_validation_details_path = f\"/content/drive/MyDrive/Saved Models/Model_{EXP_NO}/hpt_vali_m512_b32\"\n",
        "    name_of_batch_validation_details_csv = f\"{PURPOSE}_{IMG_SIZE}_b_{EFFECTIVE_BATCH_SIZE}_Validation_Metrics\"\n",
        "\n",
        "    # Save epoch wise train and val losses to monitor overfitting\n",
        "    save_progress_path = f\"/content/drive/MyDrive/Saved Models/Model_{EXP_NO}/{PURPOSE}_progress.csv\"\n",
        "\n",
        "    # Save epoch wise dice coefficient\n",
        "    save_dice_score_path = f\"/content/drive/MyDrive/Saved Models/Model_{EXP_NO}/{PURPOSE}_dice_score.csv\"\n",
        "\n",
        "    # Save bce, dice & focal losses separately after training is done\n",
        "    save_3losses_path = f\"/content/drive/MyDrive/Saved Models/Model_{EXP_NO}/{PURPOSE}_all_loss_vals.csv\"\n",
        "\n",
        "    # Save model checkpoint using early stopping class - CREATE A NEW FOLDER\n",
        "    model_checkpoint_path = f\"/content/drive/MyDrive/Saved Models/Model_{EXP_NO}/m_{IMG_SIZE}_CHECKPOINTS\"\n",
        "\n",
        "    # Save best thresholds & their dice scores - done according to Triplet Scheme of Binarization\n",
        "    save_best_thresholds_path = f\"/content/drive/MyDrive/Saved Models/Model_{EXP_NO}/{PURPOSE}_best_checkpoint_thresholds.csv\"\n",
        "\n",
        "if ACCOUNT == \"pc_aathesh\":\n",
        "    # Save batch wise comboloss during training\n",
        "    store_batch_training_details_path = f\"C:\\\\Users\\\\aathe\\\\Google Drive - Wonder Boys\\\\Saved Models\\\\Model_{EXP_NO}\\\\hpt_train_m512_b32\"\n",
        "    name_of_batch_training_details_csv = f\"{PURPOSE}_TRAIN_{IMG_SIZE}_b_{EFFECTIVE_BATCH_SIZE}_epoch_\" # epoch no. will be added to the end\n",
        "\n",
        "    # Save batch wise comboloss during validation\n",
        "    store_batch_validation_details_path = f\"C:\\\\Users\\\\aathe\\\\Google Drive - Wonder Boys\\\\Saved Models\\\\Model_{EXP_NO}\\\\hpt_vali_m512_b32\"\n",
        "    name_of_batch_validation_details_csv = f\"{PURPOSE}_{IMG_SIZE}_b_{EFFECTIVE_BATCH_SIZE}_Validation_Metrics\"\n",
        "\n",
        "    # Save epoch wise train and val losses to monitor overfitting\n",
        "    save_progress_path = f\"C:\\\\Users\\\\aathe\\\\Google Drive - Wonder Boys\\\\Saved Models\\\\Model_{EXP_NO}\\\\hpt_vali_m512_b32\\\\{PURPOSE}_progress.csv\"\n",
        "\n",
        "    # Save epoch wise dice coefficient\n",
        "    save_dice_score_path = f\"C:\\\\Users\\\\aathe\\\\Google Drive - Wonder Boys\\\\Saved Models\\\\Model_{EXP_NO}\\\\{PURPOSE}_dice_score.csv\"\n",
        "\n",
        "    # Save bce, dice & focal losses separately after training is done\n",
        "    save_3losses_path = f\"C:\\\\Users\\\\aathe\\\\Google Drive - Wonder Boys\\\\Saved Models\\\\Model_{EXP_NO}\\\\{PURPOSE}_all_loss_vals.csv\"\n",
        "\n",
        "    # Save model checkpoint using early stopping class - CREATE A NEW FOLDER\n",
        "    model_checkpoint_path = f\"C:\\\\Users\\\\aathe\\\\Google Drive - Wonder Boys\\\\Saved Models\\\\Model_{EXP_NO}\\\\m_{IMG_SIZE}_CHECKPOINTS\"\n",
        "\n",
        "    # Save best thresholds & their dice scores - done according to Triplet Scheme of Binarization\n",
        "    save_best_thresholds_path = f\"C:\\\\Users\\\\aathe\\\\Google Drive - Wonder Boys\\\\Saved Models\\\\Model_{EXP_NO}\\\\{PURPOSE}_best_checkpoint_thresholds.csv\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3A4XlffPf1jV"
      },
      "source": [
        "## Saving Config as .txt - TYPE ESSENTIAL DETAILS TO RECOVER THIS EXPERIMENT IN THE BELOW SNIPPET"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OB0Igy_Tf1jV"
      },
      "source": [
        "ESSENTIALS include current_notebook_loc_in_pc, previous_notebook_loc_in_pc, key_changes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3likqSVNf1jV",
        "outputId": "1c66c808-9cae-4c99-bf54-bdd26ef521a2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Current IST date and time is: 07-07-2024 20:35:04\n"
          ]
        }
      ],
      "source": [
        "from datetime import datetime\n",
        "import pytz\n",
        "\n",
        "# Define the IST timezone\n",
        "ist_timezone = pytz.timezone('Asia/Kolkata')\n",
        "\n",
        "# Get the current time in IST\n",
        "ist_time = datetime.now(ist_timezone)\n",
        "\n",
        "# Format the date and time to DD-MM-YYYY HH:MM:SS\n",
        "formatted_ist_time = ist_time.strftime(\"%d-%m-%Y %H:%M:%S\")\n",
        "\n",
        "# Print the formatted IST time\n",
        "print(\"Current IST date and time is:\", formatted_ist_time)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9bwNCwwVf1jV",
        "outputId": "4ccf1c5d-7a6d-47c7-dd17-7774be3dacf3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Hyper-parameters have been saved to /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Config.txt\n"
          ]
        }
      ],
      "source": [
        "# Define your hyper-parameters as a dictionary\n",
        "hyperparameters = {\n",
        "    'image_size': f\"{IMG_SIZE}x{IMG_SIZE}\",\n",
        "    'date': formatted_ist_time,\n",
        "    'account': ACCOUNT,\n",
        "    'purpose': PURPOSE,\n",
        "    'experiment_no': EXP_NO,\n",
        "    'current_notebook_loc_in_pc': \"C:/Users/nisha/Desktop/Research Project March-June/03_07_2024/Model1024/Phase_3_step_1_EXP_4.1_CosineAnnealingWarmRestarts_08_07_2024_1_PM/Model1024_HPT_EXP_ATT4_CosineAnnealingWarmRestarts_40_epochs_07_07_2024_2_PM.ipynb\",\n",
        "    'previous_notebook_loc_in_pc': \"C:/Users/nisha/Desktop/Research Project March-June/03_07_2024/Model1024/Phase_3_step_1_EXP_4_CosineAnnealingWarmRestarts_07_07_2024_8_PM/Model1024_HPT_EXP_ATT4_CosineAnnealingWarmRestarts_40_epochs_07_07_2024_2_PM.ipynb\",\n",
        "    'key_changes': \"Pretrained = ATT_4_BS_8 with 0.8294, OPTIMIZER_LOAD = True, IMG_SIZE = 1024, EARLY_STOPPING_PATIENCE = 12, epochs=50, NUM_WORKERS=12, GRADIENT_CLIPPING: 0.1, BATCH_SIZE=8, SCHEDULER='CosineAnnealingWarmRestarts', POSTIVE_PERC=0.4, LEARNING_RATE = 0.00001, L2_WEIGHT_DECAY  = 0.000005\",\n",
        "    'checkpoint_used_loc': TRAINING_MODEL_PATH,\n",
        "    'is_optimizer_loaded': OPTIMIZER_LOAD,\n",
        "    'learning_rate': LEARNING_RATE,\n",
        "    'batch_size': EFFECTIVE_BATCH_SIZE,\n",
        "    'gradient_accumulation_steps': int(EFFECTIVE_BATCH_SIZE/BATCH_SIZE),\n",
        "    'num_epochs': EPOCHS,\n",
        "    'IsSamplerUsed': USE_SAMPLER,\n",
        "    'percentage_of_positive_samples': POSTIVE_PERC,\n",
        "    'optimizer': 'Adam',\n",
        "    'loss_function': 'ComboLoss',\n",
        "    'scheduler': SCHEDULER,\n",
        "    'scheduler_params': SCHEDULER_PARAMS,\n",
        "    'L2_Regularization_weight_decay':L2_WEIGHT_DECAY,\n",
        "    'early_stopping_patience': EARLY_STOPPING_PATIENCE,\n",
        "    'is_gradient_clipping_used': GRADIENT_CLIPPING,\n",
        "    'gradient_clipping_threshold': GRADIENT_CLIPPING_THRESHOLD,\n",
        "    'model': 'U-Net with ResNet34 encoder with scSE attention',\n",
        "    'training_phase': f\"Phase - {PHASE}\",\n",
        "    'fold_ID': FOLD_ID,\n",
        "    'GPU_name': torch.cuda.get_device_name(torch.cuda.current_device()),\n",
        "    'num_workers': NUM_WORKERS,\n",
        "    'weights_given_to_loss_functions': {'bce': 3, 'dice': 1, 'focal': 4},\n",
        "    'triplet_thresholds': TRIPLET_THRESHOLDS,\n",
        "    'evaluation_metrics': EVAL_METRICS,\n",
        "}\n",
        "\n",
        "# Convert the dictionary to a formatted string\n",
        "hyperparameters_str = json.dumps(hyperparameters, indent=4)\n",
        "\n",
        "# Specify the file name\n",
        "file_name = CONFIG_FILE_LOC\n",
        "\n",
        "os.makedirs(os.path.dirname(file_name), exist_ok=True)\n",
        "\n",
        "# Write the string to a file\n",
        "with open(file_name, 'w') as file:\n",
        "    file.write(hyperparameters_str)\n",
        "\n",
        "print(f\"Hyper-parameters have been saved to {file_name}\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7xtZrBIJeLH_"
      },
      "source": [
        "## Losses"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "BjVO5UiC1z-h"
      },
      "outputs": [],
      "source": [
        "bce_losses = []\n",
        "dice_losses = []\n",
        "focal_losses = []"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "-GiUgprmeQ97"
      },
      "outputs": [],
      "source": [
        "eps = 1e-6\n",
        "\n",
        "\n",
        "def soft_dice_loss(outputs, targets, per_image=False, per_channel=False):\n",
        "    \"\"\"\n",
        "        If per_image = False, then the function calculates dice loss for a single image-mask pair.\n",
        "    \"\"\"\n",
        "    batch_size, n_channels = outputs.size(0), outputs.size(1)\n",
        "\n",
        "    eps = 1e-6\n",
        "    n_parts = 1\n",
        "    if per_image:\n",
        "        n_parts = batch_size\n",
        "    if per_channel:\n",
        "        n_parts = batch_size * n_channels\n",
        "\n",
        "    dice_target = targets.contiguous().view(n_parts, -1).float()\n",
        "    dice_output = outputs.contiguous().view(n_parts, -1)\n",
        "    intersection = torch.sum(dice_output * dice_target, dim=1)\n",
        "    union = torch.sum(dice_output, dim=1) + torch.sum(dice_target, dim=1)\n",
        "    loss = (1 - (2 * intersection + eps) / (union + eps)) # returns a tensor of size [8]\n",
        "    return loss.mean() # returns a tensor of size [1].\n",
        "\n",
        "def dice_metric(preds, trues, per_image=False, per_channel=False):\n",
        "    preds = preds.float()\n",
        "    return 1 - soft_dice_loss(preds, trues, per_image, per_channel)\n",
        "\n",
        "\n",
        "def jaccard(outputs, targets, per_image=False, non_empty=False, min_pixels=5):\n",
        "    batch_size = outputs.size()[0]\n",
        "    eps = 1e-3\n",
        "    if not per_image:\n",
        "        batch_size = 1\n",
        "    dice_target = targets.contiguous().view(batch_size, -1).float()\n",
        "    dice_output = outputs.contiguous().view(batch_size, -1)\n",
        "    target_sum = torch.sum(dice_target, dim=1)\n",
        "    intersection = torch.sum(dice_output * dice_target, dim=1)\n",
        "    losses = 1 - (intersection + eps) / (torch.sum(dice_output + dice_target, dim=1) - intersection + eps)\n",
        "    if non_empty:\n",
        "        assert per_image == True\n",
        "        non_empty_images = 0\n",
        "        sum_loss = 0\n",
        "        for i in range(batch_size):\n",
        "            if target_sum[i] > min_pixels:\n",
        "                sum_loss += losses[i]\n",
        "                non_empty_images += 1\n",
        "        if non_empty_images == 0:\n",
        "            return 0\n",
        "        else:\n",
        "            return sum_loss / non_empty_images\n",
        "\n",
        "    return losses.mean()\n",
        "\n",
        "\n",
        "class DiceLoss(nn.Module):\n",
        "    def __init__(self, weight=None, size_average=True, per_image=False):\n",
        "        super().__init__()\n",
        "        self.size_average = size_average\n",
        "        self.register_buffer('weight', weight)\n",
        "        self.per_image = per_image\n",
        "\n",
        "    def forward(self, input, target):\n",
        "        dice_loss = soft_dice_loss(input, target, per_image=self.per_image)\n",
        "        dice_losses.append(dice_loss.item())\n",
        "        return dice_loss\n",
        "\n",
        "\n",
        "class JaccardLoss(nn.Module):\n",
        "    def __init__(self, weight=None, size_average=True, per_image=False, non_empty=False, apply_sigmoid=False,\n",
        "                 min_pixels=5):\n",
        "        super().__init__()\n",
        "        self.size_average = size_average\n",
        "        self.register_buffer('weight', weight)\n",
        "        self.per_image = per_image\n",
        "        self.non_empty = non_empty\n",
        "        self.apply_sigmoid = apply_sigmoid\n",
        "        self.min_pixels = min_pixels\n",
        "\n",
        "    def forward(self, input, target):\n",
        "        if self.apply_sigmoid:\n",
        "            input = torch.sigmoid(input)\n",
        "        return jaccard(input, target, per_image=self.per_image, non_empty=self.non_empty, min_pixels=self.min_pixels)\n",
        "\n",
        "class StableBCELoss(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(StableBCELoss, self).__init__()\n",
        "\n",
        "    def forward(self, logits, target):\n",
        "        bce_loss_with_logits = nn.BCEWithLogitsLoss(reduction='mean') # mean is taken across batches\n",
        "        bce_loss = bce_loss_with_logits(logits, target)\n",
        "        bce_losses.append(bce_loss.item())\n",
        "        return bce_loss # returns a tensor of size [1]\n",
        "\n",
        "\n",
        "class ComboLoss(nn.Module):\n",
        "    def __init__(self, weights, per_image=True, channel_weights=[1, 0.5, 0.5], channel_losses=None):\n",
        "        super().__init__()\n",
        "        self.weights = weights\n",
        "        self.bce = StableBCELoss()\n",
        "        self.dice = DiceLoss(per_image=True)\n",
        "        self.jaccard = JaccardLoss(per_image=True)\n",
        "        self.lovasz = LovaszLoss(per_image=per_image)\n",
        "        self.lovasz_sigmoid = LovaszLossSigmoid(per_image=per_image)\n",
        "        self.focal = FocalLoss2d()\n",
        "        self.mapping = {'bce': self.bce,\n",
        "                        'dice': self.dice,\n",
        "                        'focal': self.focal,\n",
        "                        'jaccard': self.jaccard,\n",
        "                        'lovasz': self.lovasz,\n",
        "                        'lovasz_sigmoid': self.lovasz_sigmoid}\n",
        "        self.expect_sigmoid = {'dice', 'focal', 'jaccard', 'lovasz_sigmoid'}\n",
        "        self.per_channel = {'dice', 'jaccard', 'lovasz_sigmoid'}\n",
        "        self.values = {}\n",
        "        self.channel_weights = channel_weights\n",
        "        self.channel_losses = channel_losses\n",
        "\n",
        "    def forward(self, outputs, targets):\n",
        "        loss = 0\n",
        "        weights = self.weights\n",
        "        sigmoid_input = torch.sigmoid(outputs)\n",
        "        for k, v in weights.items():\n",
        "            if not v:\n",
        "                continue\n",
        "            val = 0\n",
        "            if k in self.per_channel:\n",
        "                channels = targets.size(1)\n",
        "                for c in range(channels):\n",
        "                    if not self.channel_losses or k in self.channel_losses[c]:\n",
        "                        val += self.channel_weights[c] * self.mapping[k](sigmoid_input[:, c, ...] if k in self.expect_sigmoid else outputs[:, c, ...],\n",
        "                                               targets[:, c, ...])\n",
        "\n",
        "            else:\n",
        "                val = self.mapping[k](sigmoid_input if k in self.expect_sigmoid else outputs, targets)\n",
        "\n",
        "            self.values[k] = val\n",
        "            loss += self.weights[k] * val\n",
        "        return loss.clamp(min=1e-5)\n",
        "\n",
        "\n",
        "def lovasz_grad(gt_sorted):\n",
        "    \"\"\"\n",
        "    Computes gradient of the Lovasz extension w.r.t sorted errors\n",
        "    See Alg. 1 in paper\n",
        "    \"\"\"\n",
        "    p = len(gt_sorted)\n",
        "    gts = gt_sorted.sum()\n",
        "    intersection = gts.float() - gt_sorted.float().cumsum(0)\n",
        "    union = gts.float() + (1 - gt_sorted).float().cumsum(0)\n",
        "    jaccard = 1. - intersection / union\n",
        "    if p > 1:  # cover 1-pixel case\n",
        "        jaccard[1:p] = jaccard[1:p] - jaccard[0:-1]\n",
        "    return jaccard\n",
        "\n",
        "\n",
        "def lovasz_hinge(logits, labels, per_image=True, ignore=None):\n",
        "    \"\"\"\n",
        "    Binary Lovasz hinge loss\n",
        "      logits: [B, H, W] Variable, logits at each pixel (between -\\infty and +\\infty)\n",
        "      labels: [B, H, W] Tensor, binary ground truth masks (0 or 1)\n",
        "      per_image: compute the loss per image instead of per batch\n",
        "      ignore: void class id\n",
        "    \"\"\"\n",
        "    if per_image:\n",
        "        loss = mean(lovasz_hinge_flat(*flatten_binary_scores(log.unsqueeze(0), lab.unsqueeze(0), ignore))\n",
        "                    for log, lab in zip(logits, labels))\n",
        "    else:\n",
        "        loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))\n",
        "    return loss\n",
        "\n",
        "\n",
        "def lovasz_hinge_flat(logits, labels):\n",
        "    \"\"\"\n",
        "    Binary Lovasz hinge loss\n",
        "      logits: [P] Variable, logits at each prediction (between -\\infty and +\\infty)\n",
        "      labels: [P] Tensor, binary ground truth labels (0 or 1)\n",
        "      ignore: label to ignore\n",
        "    \"\"\"\n",
        "    if len(labels) == 0:\n",
        "        # only void pixels, the gradients should be 0\n",
        "        return logits.sum() * 0.\n",
        "    signs = 2. * labels.float() - 1.\n",
        "    errors = (1. - logits * Variable(signs))\n",
        "    errors_sorted, perm = torch.sort(errors, dim=0, descending=True)\n",
        "    perm = perm.data\n",
        "    gt_sorted = labels[perm]\n",
        "    grad = lovasz_grad(gt_sorted)\n",
        "    loss = torch.dot(F.relu(errors_sorted), Variable(grad))\n",
        "    return loss\n",
        "\n",
        "\n",
        "def flatten_binary_scores(scores, labels, ignore=None):\n",
        "    \"\"\"\n",
        "    Flattens predictions in the batch (binary case)\n",
        "    Remove labels equal to 'ignore'\n",
        "    \"\"\"\n",
        "    scores = scores.view(-1)\n",
        "    labels = labels.view(-1)\n",
        "    if ignore is None:\n",
        "        return scores, labels\n",
        "    valid = (labels != ignore)\n",
        "    vscores = scores[valid]\n",
        "    vlabels = labels[valid]\n",
        "    return vscores, vlabels\n",
        "\n",
        "\n",
        "def lovasz_sigmoid(probas, labels, per_image=False, ignore=None):\n",
        "    \"\"\"\n",
        "    Multi-class Lovasz-Softmax loss\n",
        "      probas: [B, C, H, W] Variable, class probabilities at each prediction (between 0 and 1)\n",
        "      labels: [B, H, W] Tensor, ground truth labels (between 0 and C - 1)\n",
        "      only_present: average only on classes present in ground truth\n",
        "      per_image: compute the loss per image instead of per batch\n",
        "      ignore: void class labels\n",
        "    \"\"\"\n",
        "    if per_image:\n",
        "        loss = mean(lovasz_sigmoid_flat(*flatten_binary_scores(prob.unsqueeze(0), lab.unsqueeze(0), ignore))\n",
        "                          for prob, lab in zip(probas, labels))\n",
        "    else:\n",
        "        loss = lovasz_sigmoid_flat(*flatten_binary_scores(probas, labels, ignore))\n",
        "    return loss\n",
        "\n",
        "\n",
        "def lovasz_sigmoid_flat(probas, labels):\n",
        "    \"\"\"\n",
        "    Multi-class Lovasz-Softmax loss\n",
        "      probas: [P, C] Variable, class probabilities at each prediction (between 0 and 1)\n",
        "      labels: [P] Tensor, ground truth labels (between 0 and C - 1)\n",
        "      only_present: average only on classes present in ground truth\n",
        "    \"\"\"\n",
        "    fg = labels.float()\n",
        "    errors = (Variable(fg) - probas).abs()\n",
        "    errors_sorted, perm = torch.sort(errors, 0, descending=True)\n",
        "    perm = perm.data\n",
        "    fg_sorted = fg[perm]\n",
        "    loss = torch.dot(errors_sorted, Variable(lovasz_grad(fg_sorted)))\n",
        "    return loss\n",
        "\n",
        "def symmetric_lovasz(outputs, targets, ):\n",
        "    return (lovasz_hinge(outputs, targets) + lovasz_hinge(-outputs, 1 - targets)) / 2\n",
        "\n",
        "def mean(l, ignore_nan=False, empty=0):\n",
        "    \"\"\"\n",
        "    nanmean compatible with generators.\n",
        "    \"\"\"\n",
        "    l = iter(l)\n",
        "    if ignore_nan:\n",
        "        l = ifilterfalse(np.isnan, l)\n",
        "    try:\n",
        "        n = 1\n",
        "        acc = next(l)\n",
        "    except StopIteration:\n",
        "        if empty == 'raise':\n",
        "            raise ValueError('Empty mean')\n",
        "        return empty\n",
        "    for n, v in enumerate(l, 2):\n",
        "        acc += v\n",
        "    if n == 1:\n",
        "        return acc\n",
        "    return acc / n\n",
        "\n",
        "\n",
        "class LovaszLoss(nn.Module):\n",
        "    def __init__(self, ignore_index=255, per_image=True):\n",
        "        super().__init__()\n",
        "        self.ignore_index = ignore_index\n",
        "        self.per_image = per_image\n",
        "\n",
        "    def forward(self, outputs, targets):\n",
        "        outputs = outputs.contiguous()\n",
        "        targets = targets.contiguous()\n",
        "        return symmetric_lovasz(outputs, targets)\n",
        "\n",
        "class LovaszLossSigmoid(nn.Module):\n",
        "    def __init__(self, ignore_index=255, per_image=True):\n",
        "        super().__init__()\n",
        "        self.ignore_index = ignore_index\n",
        "        self.per_image = per_image\n",
        "\n",
        "    def forward(self, outputs, targets):\n",
        "        outputs = outputs.contiguous()\n",
        "        targets = targets.contiguous()\n",
        "        return lovasz_sigmoid(outputs, targets, per_image=self.per_image, ignore=self.ignore_index)\n",
        "\n",
        "class FocalLoss2d(nn.Module):\n",
        "    def __init__(self, alpha=0.65, gamma=2,n_parts=BATCH_SIZE):\n",
        "        super().__init__()\n",
        "        self.alpha = alpha\n",
        "        self.gamma = gamma\n",
        "        self.n_parts = n_parts\n",
        "\n",
        "    def forward(self, outputs, targets):\n",
        "        n_parts = self.n_parts\n",
        "        outputs = outputs.contiguous()\n",
        "        targets = targets.contiguous()\n",
        "        eps = 1e-6\n",
        "        # non_ignored = targets.view(n_parts, -1) != self.ignore_index\n",
        "        targets = targets.view(n_parts, -1).float()\n",
        "        outputs = outputs.view(n_parts, -1)\n",
        "        # clamp just makes sure the values of the tensor is within the given range.\n",
        "        outputs = torch.clamp(outputs, eps, 1. - eps)\n",
        "        targets = torch.clamp(targets, eps, 1. - eps)\n",
        "        \"\"\" pt = predicted probability for the true class\n",
        "        when targets = 1, pt = outputs which means pt now has the predicted probability of positive class.\n",
        "        when tagets = 0, pt = 1 - outputs which means pt has the predicted probability of negative class.\n",
        "        \"\"\"\n",
        "        pt = (1 - targets) * (1 - outputs) + targets * outputs\n",
        "        alpha_t = self.alpha * targets + (1 - self.alpha)*(1 - targets)\n",
        "        pt_proc = -(alpha_t*((1. - pt) ** self.gamma * torch.log(pt))) # torch.log is natural log\n",
        "        focal_loss = pt_proc.mean(dim=1).mean()\n",
        "        focal_losses.append(focal_loss.item())\n",
        "        return focal_loss"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A-y8ovXEK2St"
      },
      "source": [
        "## Config - 2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "g3__XL68K2St"
      },
      "outputs": [],
      "source": [
        "CRITERION        = ComboLoss(**{'weights':{'bce':3, 'dice':1, 'focal':4}})"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EqxMQpqB5JFV"
      },
      "source": [
        "# ---------------------- DL Workflow -----------------------"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BNLEHf3QK2Sv"
      },
      "source": [
        "## 1. Data -> Tensor"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9wbgxfpGK2Sv"
      },
      "source": [
        "### Create five-fold splits"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UDyy5r93K2Sv",
        "outputId": "18b373d3-c039-4adb-f5f5-a4825b131e48"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(8570, 2142)"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# single fold training for now, rerun notebook to train for multi-fold\n",
        "DF       = pd.read_csv(DATA_FRAME_PATH)\n",
        "TRAIN_DF = DF.query(f'kfold!={FOLD_ID}').reset_index(drop=True)\n",
        "VAL_DF   = DF.query(f'kfold=={FOLD_ID}').reset_index(drop=True)\n",
        "len(TRAIN_DF), len(VAL_DF)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "njKNWpqFK2Sw"
      },
      "source": [
        "### Dataset and DataLoaders"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "9ywRGbXTK2Sw"
      },
      "outputs": [],
      "source": [
        "class Dataset():\n",
        "    def __init__(self, rle_df, image_base_dir, masks_base_dir, augmentation=None, mask_augmentation=None):\n",
        "        self.df                 = rle_df\n",
        "        self.image_base_dir     = image_base_dir\n",
        "        self.masks_base_dir     = masks_base_dir\n",
        "        self.image_ids          = rle_df.ImageId.values\n",
        "        self.augmentation       = augmentation\n",
        "        self.mask_augmentation  = mask_augmentation\n",
        "\n",
        "    def __image_ids__(self):\n",
        "        print(print(self.image_ids))\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.image_ids)\n",
        "\n",
        "    def __getitem__(self, i):\n",
        "        image_id  = self.image_ids[i]\n",
        "        img_path  = os.path.join(self.image_base_dir, Path(image_id+'.png'))\n",
        "        mask_path = os.path.join(self.masks_base_dir, Path(image_id+'.png'))\n",
        "        # image is grayscale but it is loaded as RGB for compatibility with model, data augmentations, loss function, etc.\n",
        "        image     = cv2.imread(img_path, 1)\n",
        "        # mask is loaded as grayscale then converted to a binary form but has 253/255, 254/255, etc values.\n",
        "        mask      = cv2.imread(mask_path, 0)\n",
        "\n",
        "        \"\"\"Normalizing input image and mask\"\"\"\n",
        "        image = (image / 255.0).astype(np.float32)\n",
        "        mask = (mask / 255.0).astype(np.float32)\n",
        "\n",
        "        # apply augmentations - DON'T APPLY ANY TRANSFORMATION TO ALREADY NORMALIZED MASK OTHER THAN ToTensorV2\n",
        "        if self.augmentation and self.mask_augmentation:\n",
        "            img_sample = {\"image\": image}\n",
        "            img_sample = self.augmentation(**img_sample)\n",
        "            image = img_sample['image']\n",
        "            mask_sample = {\"image\": mask}\n",
        "            mask_sample = self.mask_augmentation(**mask_sample)\n",
        "            mask = mask_sample[\"image\"]\n",
        "\n",
        "        return {\n",
        "            'image': image,\n",
        "            'mask' : mask,\n",
        "        }"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "Zxv5It8ZK2Sw"
      },
      "outputs": [],
      "source": [
        "# Test transforms\n",
        "TEST_TFMS = albu.Compose([\n",
        "    albu.Resize(height=IMG_SIZE, width=IMG_SIZE, always_apply=True, p=1),\n",
        "    albu.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n",
        "    ToTensorV2(),\n",
        "],\n",
        "    is_check_shapes=True,\n",
        "    p=1.0,\n",
        ")\n",
        "\n",
        "# New Train Transforms - Aggressive Augmentations to avoid overfitting.\n",
        "TFMS =  albu.Compose(\n",
        "    [\n",
        "        albu.OneOf([\n",
        "            albu.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.5),\n",
        "            albu.RandomGamma(gamma_limit=(80, 120), p=0.5),\n",
        "        ], p=0.3),\n",
        "        albu.OneOf([\n",
        "            albu.ElasticTransform(alpha=120, sigma=6.0, alpha_affine=3.6, p=0.5),\n",
        "            albu.GridDistortion(num_steps=5, distort_limit=(-0.3, 0.3), p=0.5),\n",
        "            albu.OpticalDistortion(distort_limit=(-2, 2), shift_limit=(-0.5, 0.5), p=0.5),\n",
        "        ], p=0.3),\n",
        "        albu.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=45, p=0.5),\n",
        "        albu.Resize(height=IMG_SIZE, width=IMG_SIZE, p=1.0),\n",
        "        albu.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], p=1.0),\n",
        "        ToTensorV2(),\n",
        "    ],\n",
        "    p=1.0\n",
        ")\n",
        "\n",
        "mask_transform = albu.Compose([\n",
        "    albu.Resize(height=IMG_SIZE, width=IMG_SIZE, p=1.0),\n",
        "    ToTensorV2()  # Directly convert binary mask to tensor without normalization\n",
        "])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "4DKpUR1QK2Sw"
      },
      "outputs": [],
      "source": [
        "# train dataset\n",
        "train_dataset = Dataset(TRAIN_DF, TRAIN_IMG_DIR, TRAIN_LBL_DIR, TFMS, mask_transform)\n",
        "val_dataset   = Dataset(VAL_DF, TRAIN_IMG_DIR, TRAIN_LBL_DIR, TEST_TFMS, mask_transform)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7zno_0Q4eVHv",
        "outputId": "32227551-9c2b-4fce-f541-cbc5472d0b7a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "8570"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_dataset.__len__()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6-dcysMmK2Sw"
      },
      "source": [
        "### Sampler"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "Qx49nrt4K2Sw"
      },
      "outputs": [],
      "source": [
        "class PneumoSampler(Sampler):\n",
        "    def __init__(self, train_df, positive_perc=0.8):\n",
        "        assert positive_perc > 0, 'percentage of positive pneumothorax images must be greater then zero'\n",
        "        self.train_df = train_df\n",
        "        self.positive_perc = positive_perc\n",
        "        self.positive_idxs = self.train_df.query('has_mask==1').index.values\n",
        "        self.negative_idxs = self.train_df.query('has_mask!=1').index.values\n",
        "        self.n_positive = len(self.positive_idxs)\n",
        "        self.n_negative = int(self.n_positive * (1 - self.positive_perc) / self.positive_perc)\n",
        "\n",
        "    def __iter__(self):\n",
        "        negative_sample = np.random.choice(self.negative_idxs, size=self.n_negative)\n",
        "        shuffled = np.random.permutation(np.hstack((negative_sample, self.positive_idxs)))\n",
        "        return iter(shuffled.tolist())\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.n_positive + self.n_negative"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "kzJerUgtK2Sx"
      },
      "outputs": [],
      "source": [
        "SAMPLER = PneumoSampler(TRAIN_DF, positive_perc=POSTIVE_PERC)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Asd-jMF9K2Sx"
      },
      "source": [
        "### DataLoaders"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "Saav3bv6K2Sx"
      },
      "outputs": [],
      "source": [
        "# dataloaders\n",
        "train_dataloader = DataLoader(train_dataset, TRAIN_BATCH_SIZE,\n",
        "                              shuffle=True if not USE_SAMPLER else False,\n",
        "                              num_workers=NUM_WORKERS,\n",
        "                              sampler=SAMPLER if USE_SAMPLER else None)\n",
        "val_dataloader   = DataLoader(val_dataset, VALID_BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u9nGVmHlj6z5",
        "outputId": "5f204f01-008b-4d2b-e45c-b508ddd43b95"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Dataloaders: (<torch.utils.data.dataloader.DataLoader object at 0x7f6c0e26b370>, <torch.utils.data.dataloader.DataLoader object at 0x7f6c0e2691e0>)\n",
            "Length of train dataloader: 595 batches of 8\n",
            "length of validation dataloader: 268 batches of 8\n"
          ]
        }
      ],
      "source": [
        "print(f\"Dataloaders: {train_dataloader , val_dataloader}\")\n",
        "print(f\"Length of train dataloader: {len(train_dataloader)} batches of {BATCH_SIZE}\")\n",
        "print(f\"length of validation dataloader: {len(val_dataloader)} batches of {BATCH_SIZE}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0jmBiDQJwFTN"
      },
      "source": [
        "for batch_index, data in enumerate(train_dataloader):\n",
        "    for z in range(3):\n",
        "        print(\"Image: \", data['image'].shape)\n",
        "        print(\"Mask: \", data['mask'].shape)\n",
        "        print(\"data['mask'].unsqueeze(1)\", data['mask'].unsqueeze(1).shape)\n",
        "    break\n",
        "\n",
        "Output\n",
        "Image:  torch.Size([8, 3, 512, 512])\n",
        "Mask:  torch.Size([8, 1, 512, 512])\n",
        "data['mask'].unsqueeze(1) torch.Size([8, 1, 1, 512, 512])\n",
        "Image:  torch.Size([8, 3, 512, 512])\n",
        "Mask:  torch.Size([8, 1, 512, 512])\n",
        "data['mask'].unsqueeze(1) torch.Size([8, 1, 1, 512, 512])\n",
        "Image:  torch.Size([8, 3, 512, 512])\n",
        "Mask:  torch.Size([8, 1, 512, 512])\n",
        "data['mask'].unsqueeze(1) torch.Size([8, 1, 1, 512, 512])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MIIdw0tLK2S4"
      },
      "source": [
        "## 2. Model Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "noi6vEbbK2S4",
        "outputId": "1d236495-731e-4c60-b1f8-8f126820dcd7"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Downloading: \"https://download.pytorch.org/models/resnet34-333f7ec4.pth\" to /root/.cache/torch/hub/checkpoints/resnet34-333f7ec4.pth\n",
            "100%|██████████| 83.3M/83.3M [00:00<00:00, 424MB/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Unet(\n",
            "  (encoder): ResNetEncoder(\n",
            "    (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
            "    (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "    (relu): ReLU(inplace=True)\n",
            "    (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
            "    (layer1): Sequential(\n",
            "      (0): BasicBlock(\n",
            "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "      (1): BasicBlock(\n",
            "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "      (2): BasicBlock(\n",
            "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "    )\n",
            "    (layer2): Sequential(\n",
            "      (0): BasicBlock(\n",
            "        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (downsample): Sequential(\n",
            "          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
            "          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        )\n",
            "      )\n",
            "      (1): BasicBlock(\n",
            "        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "      (2): BasicBlock(\n",
            "        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "      (3): BasicBlock(\n",
            "        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "    )\n",
            "    (layer3): Sequential(\n",
            "      (0): BasicBlock(\n",
            "        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (downsample): Sequential(\n",
            "          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
            "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        )\n",
            "      )\n",
            "      (1): BasicBlock(\n",
            "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "      (2): BasicBlock(\n",
            "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "      (3): BasicBlock(\n",
            "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "      (4): BasicBlock(\n",
            "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "      (5): BasicBlock(\n",
            "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "    )\n",
            "    (layer4): Sequential(\n",
            "      (0): BasicBlock(\n",
            "        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (downsample): Sequential(\n",
            "          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
            "          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        )\n",
            "      )\n",
            "      (1): BasicBlock(\n",
            "        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "      (2): BasicBlock(\n",
            "        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "        (relu): ReLU(inplace=True)\n",
            "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (decoder): UnetDecoder(\n",
            "    (center): Identity()\n",
            "    (blocks): ModuleList(\n",
            "      (0): DecoderBlock(\n",
            "        (conv1): Conv2dReLU(\n",
            "          (0): Conv2d(768, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "          (2): ReLU(inplace=True)\n",
            "        )\n",
            "        (attention1): Attention(\n",
            "          (attention): SCSEModule(\n",
            "            (cSE): Sequential(\n",
            "              (0): AdaptiveAvgPool2d(output_size=1)\n",
            "              (1): Conv2d(768, 48, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (2): ReLU(inplace=True)\n",
            "              (3): Conv2d(48, 768, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (4): Sigmoid()\n",
            "            )\n",
            "            (sSE): Sequential(\n",
            "              (0): Conv2d(768, 1, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (1): Sigmoid()\n",
            "            )\n",
            "          )\n",
            "        )\n",
            "        (conv2): Conv2dReLU(\n",
            "          (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "          (2): ReLU(inplace=True)\n",
            "        )\n",
            "        (attention2): Attention(\n",
            "          (attention): SCSEModule(\n",
            "            (cSE): Sequential(\n",
            "              (0): AdaptiveAvgPool2d(output_size=1)\n",
            "              (1): Conv2d(256, 16, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (2): ReLU(inplace=True)\n",
            "              (3): Conv2d(16, 256, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (4): Sigmoid()\n",
            "            )\n",
            "            (sSE): Sequential(\n",
            "              (0): Conv2d(256, 1, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (1): Sigmoid()\n",
            "            )\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (1): DecoderBlock(\n",
            "        (conv1): Conv2dReLU(\n",
            "          (0): Conv2d(384, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "          (2): ReLU(inplace=True)\n",
            "        )\n",
            "        (attention1): Attention(\n",
            "          (attention): SCSEModule(\n",
            "            (cSE): Sequential(\n",
            "              (0): AdaptiveAvgPool2d(output_size=1)\n",
            "              (1): Conv2d(384, 24, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (2): ReLU(inplace=True)\n",
            "              (3): Conv2d(24, 384, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (4): Sigmoid()\n",
            "            )\n",
            "            (sSE): Sequential(\n",
            "              (0): Conv2d(384, 1, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (1): Sigmoid()\n",
            "            )\n",
            "          )\n",
            "        )\n",
            "        (conv2): Conv2dReLU(\n",
            "          (0): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "          (2): ReLU(inplace=True)\n",
            "        )\n",
            "        (attention2): Attention(\n",
            "          (attention): SCSEModule(\n",
            "            (cSE): Sequential(\n",
            "              (0): AdaptiveAvgPool2d(output_size=1)\n",
            "              (1): Conv2d(128, 8, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (2): ReLU(inplace=True)\n",
            "              (3): Conv2d(8, 128, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (4): Sigmoid()\n",
            "            )\n",
            "            (sSE): Sequential(\n",
            "              (0): Conv2d(128, 1, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (1): Sigmoid()\n",
            "            )\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (2): DecoderBlock(\n",
            "        (conv1): Conv2dReLU(\n",
            "          (0): Conv2d(192, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "          (2): ReLU(inplace=True)\n",
            "        )\n",
            "        (attention1): Attention(\n",
            "          (attention): SCSEModule(\n",
            "            (cSE): Sequential(\n",
            "              (0): AdaptiveAvgPool2d(output_size=1)\n",
            "              (1): Conv2d(192, 12, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (2): ReLU(inplace=True)\n",
            "              (3): Conv2d(12, 192, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (4): Sigmoid()\n",
            "            )\n",
            "            (sSE): Sequential(\n",
            "              (0): Conv2d(192, 1, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (1): Sigmoid()\n",
            "            )\n",
            "          )\n",
            "        )\n",
            "        (conv2): Conv2dReLU(\n",
            "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "          (2): ReLU(inplace=True)\n",
            "        )\n",
            "        (attention2): Attention(\n",
            "          (attention): SCSEModule(\n",
            "            (cSE): Sequential(\n",
            "              (0): AdaptiveAvgPool2d(output_size=1)\n",
            "              (1): Conv2d(64, 4, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (2): ReLU(inplace=True)\n",
            "              (3): Conv2d(4, 64, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (4): Sigmoid()\n",
            "            )\n",
            "            (sSE): Sequential(\n",
            "              (0): Conv2d(64, 1, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (1): Sigmoid()\n",
            "            )\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (3): DecoderBlock(\n",
            "        (conv1): Conv2dReLU(\n",
            "          (0): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "          (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "          (2): ReLU(inplace=True)\n",
            "        )\n",
            "        (attention1): Attention(\n",
            "          (attention): SCSEModule(\n",
            "            (cSE): Sequential(\n",
            "              (0): AdaptiveAvgPool2d(output_size=1)\n",
            "              (1): Conv2d(128, 8, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (2): ReLU(inplace=True)\n",
            "              (3): Conv2d(8, 128, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (4): Sigmoid()\n",
            "            )\n",
            "            (sSE): Sequential(\n",
            "              (0): Conv2d(128, 1, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (1): Sigmoid()\n",
            "            )\n",
            "          )\n",
            "        )\n",
            "        (conv2): Conv2dReLU(\n",
            "          (0): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "          (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "          (2): ReLU(inplace=True)\n",
            "        )\n",
            "        (attention2): Attention(\n",
            "          (attention): SCSEModule(\n",
            "            (cSE): Sequential(\n",
            "              (0): AdaptiveAvgPool2d(output_size=1)\n",
            "              (1): Conv2d(32, 2, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (2): ReLU(inplace=True)\n",
            "              (3): Conv2d(2, 32, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (4): Sigmoid()\n",
            "            )\n",
            "            (sSE): Sequential(\n",
            "              (0): Conv2d(32, 1, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (1): Sigmoid()\n",
            "            )\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "      (4): DecoderBlock(\n",
            "        (conv1): Conv2dReLU(\n",
            "          (0): Conv2d(32, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "          (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "          (2): ReLU(inplace=True)\n",
            "        )\n",
            "        (attention1): Attention(\n",
            "          (attention): SCSEModule(\n",
            "            (cSE): Sequential(\n",
            "              (0): AdaptiveAvgPool2d(output_size=1)\n",
            "              (1): Conv2d(32, 2, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (2): ReLU(inplace=True)\n",
            "              (3): Conv2d(2, 32, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (4): Sigmoid()\n",
            "            )\n",
            "            (sSE): Sequential(\n",
            "              (0): Conv2d(32, 1, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (1): Sigmoid()\n",
            "            )\n",
            "          )\n",
            "        )\n",
            "        (conv2): Conv2dReLU(\n",
            "          (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "          (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "          (2): ReLU(inplace=True)\n",
            "        )\n",
            "        (attention2): Attention(\n",
            "          (attention): SCSEModule(\n",
            "            (cSE): Sequential(\n",
            "              (0): AdaptiveAvgPool2d(output_size=1)\n",
            "              (1): Conv2d(16, 1, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (2): ReLU(inplace=True)\n",
            "              (3): Conv2d(1, 16, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (4): Sigmoid()\n",
            "            )\n",
            "            (sSE): Sequential(\n",
            "              (0): Conv2d(16, 1, kernel_size=(1, 1), stride=(1, 1))\n",
            "              (1): Sigmoid()\n",
            "            )\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (segmentation_head): SegmentationHead(\n",
            "    (0): Conv2d(16, 1, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (1): Identity()\n",
            "    (2): Activation(\n",
            "      (activation): Identity()\n",
            "    )\n",
            "  )\n",
            ")\n"
          ]
        }
      ],
      "source": [
        "import segmentation_models_pytorch as smp\n",
        "\n",
        "# Create a U-Net model with a specified encoder\n",
        "model = smp.Unet(encoder_name=\"resnet34\",        # Choose encoder, e.g., resnet34 or any other encoder\n",
        "                 encoder_weights=\"imagenet\",     # Use pre-trained weights for the encoder\n",
        "                 in_channels=3,                  # Model input channels (1 for grayscale, 3 for RGB)\n",
        "                 classes=1,\n",
        "                 decoder_attention_type='scse')                      # Model output channels (number of classes)\n",
        "# model = smp.Unet(encoder_name=\"resnet34\",        # Choose encoder, e.g., resnet34 or any other encoder\n",
        "#                  in_channels=3,                  # Model input channels (1 for grayscale, 3 for RGB)\n",
        "#                  classes=1)                      # Model output channels (number of classes)\n",
        "\n",
        "# Print the model architecture\n",
        "print(model)\n",
        "# Input has 3 color channels [RGB] & Output has 1 color channel\n",
        "# Format NCHW - no. of samples in a mini-batch, no. of color channels, height and width."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vzze2VhIK2S5"
      },
      "source": [
        "### Early Stopping"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "TbTO94ePK2S5"
      },
      "outputs": [],
      "source": [
        "class EarlyStopping:\n",
        "    def __init__(self, patience=7, mode=\"max\", delta=0.0001):\n",
        "        self.patience = patience\n",
        "        self.counter = 0\n",
        "        self.mode = mode\n",
        "        self.best_score = None\n",
        "        self.early_stop = False\n",
        "        self.delta = delta\n",
        "        if self.mode == \"min\":\n",
        "            self.val_score = np.Inf\n",
        "        else:\n",
        "            self.val_score = -np.Inf\n",
        "\n",
        "    def __call__(self, epoch, epoch_score, model, optimizer, loss, model_path):\n",
        "        if self.mode == \"min\":\n",
        "            score = -1.0 * epoch_score\n",
        "        else:\n",
        "            score = np.copy(epoch_score)\n",
        "\n",
        "        if self.best_score is None:\n",
        "            self.best_score = score\n",
        "            self.save_checkpoint(epoch, epoch_score, model, optimizer, loss, model_path)\n",
        "        elif score < self.best_score + self.delta:\n",
        "            self.counter += 1\n",
        "            print(\n",
        "                \"EarlyStopping counter: {} out of {}\".format(\n",
        "                    self.counter, self.patience\n",
        "                )\n",
        "            )\n",
        "            if self.counter >= self.patience:\n",
        "                self.early_stop = True\n",
        "        else:\n",
        "            self.best_score = score\n",
        "            self.save_checkpoint(epoch, epoch_score, model, optimizer, loss, model_path)\n",
        "            self.counter = 0\n",
        "\n",
        "    def save_checkpoint(self, epoch, epoch_score, model, optimizer, loss, model_path):\n",
        "        model_path = Path(model_path)\n",
        "        parent = model_path.parent\n",
        "        os.makedirs(parent, exist_ok=True)\n",
        "        if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:\n",
        "            print(\n",
        "                \"Validation score improved ({} --> {}). Model saved at {}!\".format(\n",
        "                    self.val_score, epoch_score, model_path\n",
        "                )\n",
        "            )\n",
        "            torch.save({\"Epoch\": epoch, \"Train Loss\": loss, \"Validation Dice Score\": self.val_score, \"model_state_dict\": model.state_dict(), \"optimizer_state_dict\": optimizer.state_dict()}, model_path)\n",
        "            BEST_MODEL_PATH = model_path\n",
        "            print(f\"Checkpoint saved on epoch - {epoch} with dice score - {epoch_score}\")\n",
        "        self.val_score = epoch_score"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kOlM9jTEK2S5"
      },
      "source": [
        "### train_one_epoch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "_amog1GHK2S5"
      },
      "outputs": [],
      "source": [
        "class AverageMeter:\n",
        "    def __init__(self):\n",
        "        self.val = 0\n",
        "        self.avg = 0\n",
        "        self.sum = 0\n",
        "        self.count = 0\n",
        "\n",
        "    def reset(self):\n",
        "        self.val = 0\n",
        "        self.avg = 0\n",
        "        self.sum = 0\n",
        "        self.count = 0\n",
        "\n",
        "    def update(self, val, n=1):\n",
        "        self.val = val\n",
        "        self.sum += val * n\n",
        "        self.count += n\n",
        "        self.avg = self.sum / self.count"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "xbn3pKTaK2S5"
      },
      "outputs": [],
      "source": [
        "def train_one_epoch(train_loader, model, optimizer, loss_fn, accumulation_steps=int(EFFECTIVE_BATCH_SIZE/BATCH_SIZE), device='cuda'):\n",
        "    losses = AverageMeter()\n",
        "\n",
        "    # Lists to store batch-to-batch progress details within the epoch while training\n",
        "    batch_count_train = []\n",
        "    batch_train_loss = []\n",
        "\n",
        "    model = model.to(device)\n",
        "    model.train()\n",
        "    if accumulation_steps > 1:\n",
        "      optimizer.zero_grad()\n",
        "    tk0 = tqdm(train_loader, total=len(train_loader))\n",
        "    for b_idx, data in enumerate(tk0):\n",
        "      # print(data['image'].shape) -> torch.Size([8, 3, 512, 512])\n",
        "      # print(data['mask'].shape) -> torch.Size([8, 1, 512, 512])\n",
        "      if (b_idx + 1) % accumulation_steps == 0:\n",
        "        batch_count_train.append(b_idx)\n",
        "\n",
        "      # moves image tensor and mask tensor to gpu\n",
        "      for key, value in data.items():\n",
        "        data[key] = value.to(\"cuda\")\n",
        "\n",
        "      if accumulation_steps == 1 and b_idx == 0:\n",
        "        optimizer.zero_grad()\n",
        "      out  = model(data['image']) # out.shape = torch.Size([8, 1, 512, 512])\n",
        "      loss = loss_fn(out, data['mask'].float()) # mask.shape = torch.Size([8, 1, 512, 512])\n",
        "      with torch.set_grad_enabled(True):\n",
        "        loss.backward()\n",
        "        if (b_idx + 1) % accumulation_steps == 0:\n",
        "          if GRADIENT_CLIPPING:\n",
        "            clip_grad_norm_(model.parameters(), GRADIENT_CLIPPING_THRESHOLD)\n",
        "          optimizer.step()\n",
        "          optimizer.zero_grad()\n",
        "      losses.update(loss.item(), train_loader.batch_size)\n",
        "      if (b_idx + 1) % accumulation_steps == 0:\n",
        "        batch_train_loss.append(loss.item())\n",
        "      tk0.set_postfix(loss=losses.avg, learning_rate=optimizer.param_groups[0]['lr'])\n",
        "    return losses.avg, batch_count_train, batch_train_loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "id": "QCUdPtqF-unk"
      },
      "outputs": [],
      "source": [
        "criterion = CRITERION\n",
        "es = EarlyStopping(patience=EARLY_STOPPING_PATIENCE, mode='max')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FKL6S336-unk",
        "outputId": "6cf126b9-50fb-4cff-cc16-0e229f0816c3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "ComboLoss(\n",
              "  (bce): StableBCELoss()\n",
              "  (dice): DiceLoss()\n",
              "  (jaccard): JaccardLoss()\n",
              "  (lovasz): LovaszLoss()\n",
              "  (lovasz_sigmoid): LovaszLossSigmoid()\n",
              "  (focal): FocalLoss2d()\n",
              ")"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "criterion"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K136b3eqK2S5"
      },
      "source": [
        "### Validation Functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N1RY2NxC-unk"
      },
      "source": [
        "#### Mask Binarizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "id": "V4my02Uf-unk"
      },
      "outputs": [],
      "source": [
        "class MaskBinarization():\n",
        "    def __init__(self):\n",
        "        self.thresholds = 0.5\n",
        "    def transform(self, predicted):\n",
        "        yield predicted > self.thresholds\n",
        "\n",
        "class SimpleMaskBinarization(MaskBinarization):\n",
        "    def __init__(self, score_thresholds):\n",
        "        super().__init__()\n",
        "        self.thresholds = score_thresholds\n",
        "    def transform(self, predicted):\n",
        "        for thr in self.thresholds:\n",
        "            yield predicted > thr\n",
        "\n",
        "class DupletMaskBinarization(MaskBinarization):\n",
        "    def __init__(self, duplets, with_channels=True):\n",
        "        super().__init__()\n",
        "        self.thresholds = duplets\n",
        "        self.dims = (2,3) if with_channels else (1,2)\n",
        "    def transform(self, predicted):\n",
        "        for score_threshold, area_threshold in self.thresholds:\n",
        "            mask = predicted > score_threshold\n",
        "            mask[mask.sum(dim=self.dims) < area_threshold] = 0\n",
        "            yield mask\n",
        "\n",
        "class TripletMaskBinarization(MaskBinarization):\n",
        "    def __init__(self, triplets, with_channels=True):\n",
        "        super().__init__()\n",
        "        self.thresholds = triplets\n",
        "        self.dims = (2,3) if with_channels else (1,2) # dims should be HxW, basically it should ignore batch_size and no_of_channels in the general format of BxCxHxW\n",
        "    def transform(self, predicted):\n",
        "        for top_score_threshold, area_threshold, bottom_score_threshold in self.thresholds:\n",
        "            clf_mask = (predicted > top_score_threshold).float()\n",
        "            pred_mask = (predicted > bottom_score_threshold).float()\n",
        "            pred_mask[clf_mask.sum(dim=self.dims) < area_threshold] = 0\n",
        "            yield pred_mask"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "joGnfZgw-unl"
      },
      "source": [
        "#### Metric used in validation and evaluate function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "id": "zlWOcwQdK2S5"
      },
      "outputs": [],
      "source": [
        "def metric(probability, truth):\n",
        "    if probability.shape[0] == truth.shape[0]: # checking for batch size mismatches in the code for image & mask\n",
        "        batch_size = probability.shape[0]\n",
        "    with torch.no_grad():\n",
        "        probability = probability.view(batch_size, -1) # probability's size is [8, 1*512*512]\n",
        "        truth = truth.view(batch_size, -1)             # truth's size is [8, 1*512*512]\n",
        "        assert(probability.shape == truth.shape)\n",
        "\n",
        "        p = probability.float() # prob_preds already comes in binarized.\n",
        "        t = (truth > 0.5).float()\n",
        "\n",
        "        t_sum = t.sum(-1) # t_sum size is 8 # Each value in the vector represents the sum of all pixels in one mask\n",
        "        p_sum = p.sum(-1) # p_sum size is 8 # Each value in the vector represents the sum of all elements in one pred_probs\n",
        "        neg_index = torch.nonzero(t_sum == 0) # indices of masks which are negative.\n",
        "        pos_index = torch.nonzero(t_sum >= 1) # indices of masks which are positive.\n",
        "\n",
        "        dice_neg = (p_sum == 0).float() # tensor of size 8\n",
        "        \"\"\"\n",
        "        if t_sum = torch.tensor([0.0, 1000.0, 0.0, 600.0, 720.0, 420.0, 0.0, 0.0]), then dice_neg = tensor([1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0])\n",
        "        \"\"\"\n",
        "        dice_pos = 2 * (p*t).sum(-1)/((p+t).sum(-1)) # tensor of size 8\n",
        "\n",
        "        dice_neg = dice_neg[neg_index] # selects elements of dice_neg acc to the indices in neg_index, it can have more than one element.\n",
        "        dice_pos = dice_pos[pos_index] # similar to the above code line.\n",
        "        dice = torch.cat([dice_pos, dice_neg])\n",
        "\n",
        "        num_neg = len(neg_index) # no. of negative masks in a batch\n",
        "        num_pos = len(pos_index) # no. of positive masks in a batch\n",
        "\n",
        "    return dice.mean()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "id": "u3ovO3f9XHQt"
      },
      "outputs": [],
      "source": [
        "# Function to save the data dynamically\n",
        "def save_best_thresholds(epoch, b_idx, best_metric, best_threshold, filepath=Path(save_best_thresholds_path)):\n",
        "    # Create a dictionary of lists\n",
        "    data = {\n",
        "        'epoch': epoch,\n",
        "        'batch_number': b_idx,\n",
        "        'best_metric': best_metric,\n",
        "        'best_threshold': best_threshold\n",
        "    }\n",
        "\n",
        "    # Create a DataFrame\n",
        "    df = pd.DataFrame(data)\n",
        "\n",
        "    # Save DataFrame to CSV (mode='w' to write from scratch each time, mode='a' to append)\n",
        "    df.to_csv(filepath, index=False, mode='w')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "7opE5TuMRCvg"
      },
      "outputs": [],
      "source": [
        "epoch_count_thr = []\n",
        "batch_indices = []\n",
        "best_metrics_list = []\n",
        "best_thresholds_list = []"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "id": "ww6ftfKmK2S5"
      },
      "outputs": [],
      "source": [
        "def evaluate(valid_loader, model, epoch, device=DEVICE, metric=metric, loss_fn=criterion):\n",
        "    losses = AverageMeter()\n",
        "    combolosses = AverageMeter()\n",
        "    metrics = defaultdict(float)\n",
        "    # Lists to store batch-to-batch progress details within the epoch while training\n",
        "    batch_count_val = []\n",
        "    batch_val_loss_values = []\n",
        "    model = model.to(device)\n",
        "    model.eval()\n",
        "    tk0 = tqdm(valid_loader, total=len(valid_loader))\n",
        "    with torch.inference_mode():\n",
        "        for b_idx, data in enumerate(tk0):\n",
        "          batch_count_val.append(b_idx)\n",
        "          for key, value in data.items():\n",
        "            data[key] = value.to(device)\n",
        "          out   = model(data['image']) # Raw model output\n",
        "          loss = loss_fn(out, data['mask']).cpu()\n",
        "          pred_probs   = torch.sigmoid(out) # Prediction probabilities, this is already done in ComboLoss's forward function.\n",
        "          # dice and jaccard expect prediction probabilities whereas bce expects logits (raw model outputs). It's mentioned inside the class too.\n",
        "          # This is a mistake, you can't pass prediction probability into a evaluation metric like this.\n",
        "          # You must pass the mask cuz u are comparing the actual mask(target) and the predicted mask not predicted probabilities.\n",
        "          # So, 1st apply mask binarization to the prediction probabilities.\n",
        "          binarizer_fn = TripletMaskBinarization(TRIPLET_THRESHOLDS)\n",
        "          mask_generator = binarizer_fn.transform(pred_probs)\n",
        "          used_thresholds = binarizer_fn.thresholds\n",
        "          # ------------------------------------- Tests Required ---------------------------------------------------------\n",
        "          for current_thr, current_mask in zip(used_thresholds, mask_generator):\n",
        "              current_metric = metric(current_mask, data['mask']).item()\n",
        "              current_thr = tuple(current_thr)\n",
        "              metrics[current_thr] = (metrics[current_thr] * b_idx + current_metric) / (b_idx + 1)\n",
        "\n",
        "          best_threshold = max(metrics, key=metrics.get)\n",
        "          best_metric = metrics[best_threshold]\n",
        "          tk0.set_description('score: {:.5f} on {}'.format(best_metric, best_threshold))\n",
        "          dice = best_metric\n",
        "\n",
        "          epoch_count_thr.append(epoch)\n",
        "          batch_indices.append(b_idx)\n",
        "          best_metrics_list.append(best_metric)\n",
        "          best_thresholds_list.append(best_threshold)\n",
        "          save_best_thresholds(epoch_count_thr, batch_indices, best_metrics_list, best_thresholds_list)\n",
        "\n",
        "          # if .item() is used then .cpu() is NOT required. .item() will itself return a float value.\n",
        "          losses.update(dice, valid_loader.batch_size)\n",
        "          combolosses.update(loss.item(), valid_loader.batch_size)\n",
        "          batch_val_loss_values.append(loss.item())\n",
        "          tk0.set_postfix(dice_score=losses.avg, val_loss=combolosses.avg)\n",
        "    return losses.avg, batch_count_val, batch_val_loss_values, combolosses.avg"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mE7TO5J9XHQt"
      },
      "source": [
        "### Optimizer & Scheduler"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "id": "Fn0XAwlyK2S6"
      },
      "outputs": [],
      "source": [
        "if PRETRAINED:\n",
        "  checkpoint = torch.load(TRAINING_MODEL_PATH)\n",
        "  model.to(DEVICE)\n",
        "  model.load_state_dict(checkpoint[\"model_state_dict\"])\n",
        "  optimizer = torch.optim.Adam(model.parameters(), lr= LEARNING_RATE, weight_decay= L2_WEIGHT_DECAY)\n",
        "  if OPTIMIZER_LOAD:\n",
        "    optimizer.load_state_dict(checkpoint[\"optimizer_state_dict\"])\n",
        "\n",
        "  # Manually set the new learning rate for this stage of training as loading optimizer's state dict will\n",
        "  # load parameters that was there while saving the previous checkpoint but loading the optimizer's state dict is\n",
        "  # crucial\n",
        "  for param_group in optimizer.param_groups:\n",
        "    param_group['lr'] = LEARNING_RATE\n",
        "\n",
        "  if SCHEDULER == \"ReduceLROnPlateau\":\n",
        "    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=SCHEDULER_PARAMS[\"factor\"], patience=SCHEDULER_PARAMS[\"patience\"], threshold=SCHEDULER_PARAMS[\"threshold\"], min_lr=SCHEDULER_PARAMS[\"min_lr\"])\n",
        "  elif SCHEDULER == \"CosineAnnealingWarmRestarts\":\n",
        "    scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=SCHEDULER_PARAMS[\"T_0\"], T_mult=SCHEDULER_PARAMS[\"T_mult\"], eta_min=SCHEDULER_PARAMS[\"eta_min\"])\n",
        "  elif SCHEDULER == \"CosineAnnealingLR\":\n",
        "    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=SCHEDULER_PARAMS[\"T_max\"], eta_min=SCHEDULER_PARAMS[\"eta_min\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "id": "gK69cmOlRCvg"
      },
      "outputs": [],
      "source": [
        "if not PRETRAINED:\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr= LEARNING_RATE, weight_decay= L2_WEIGHT_DECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=SCHEDULER_PARAMS[\"factor\"], patience=SCHEDULER_PARAMS[\"patience\"], threshold=SCHEDULER_PARAMS[\"threshold\"], min_lr=SCHEDULER_PARAMS[\"min_lr\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LqlFcFDSRCvh",
        "outputId": "637f07b0-7eac-44f6-b44f-c88d30947096"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "New learning rate: 1e-05\n"
          ]
        }
      ],
      "source": [
        "for param_group in optimizer.param_groups:\n",
        "    print(f\"New learning rate: {param_group['lr']}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ig7E3afMOi-6",
        "outputId": "263ea848-38e9-442d-b82b-0f24380e38e0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Initial learning rate of scheduler: 1e-05\n"
          ]
        }
      ],
      "source": [
        "initial_lr_scheduler = scheduler.base_lrs[0]  # Assuming a single learning rate for all parameters\n",
        "print(\"Initial learning rate of scheduler:\", initial_lr_scheduler)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RtHHjC45vsqr"
      },
      "source": [
        "### Training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LcACWXgAXHQu"
      },
      "source": [
        "#### Saving Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {
        "id": "AzIGIAe2FmJQ"
      },
      "outputs": [],
      "source": [
        "def store_batch_training_details(epoch, batch_count_train, batch_train_loss):\n",
        "  # Directory where you want to save the CSV file\n",
        "  directory = store_batch_training_details_path\n",
        "\n",
        "  # Ensure the directory exists\n",
        "  os.makedirs(directory, exist_ok=True)\n",
        "\n",
        "  # File path\n",
        "  file_path = os.path.join(directory, name_of_batch_training_details_csv + f\"{epoch}.csv\")\n",
        "\n",
        "  # Write to CSV file\n",
        "  with open(file_path, \"w\", newline=\"\") as file:\n",
        "      writer = csv.writer(file)\n",
        "      # Write header\n",
        "      writer.writerow([\"Batch Count Train\", \"Batch Train Loss\"])\n",
        "      # Write data rows\n",
        "      for batch_count, train_loss in zip(batch_count_train, batch_train_loss):\n",
        "          writer.writerow([batch_count, train_loss])\n",
        "\n",
        "  print(f\"CSV file for epoch-{epoch}has been created at: {file_path}\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "id": "f5-0shprvsqr"
      },
      "outputs": [],
      "source": [
        "def store_batch_validation_details(epoch, batch_count_val, batch_val_score_values):\n",
        "  # Directory where you want to save the CSV file\n",
        "  directory = store_batch_validation_details_path\n",
        "\n",
        "  # Ensure the directory exists\n",
        "  os.makedirs(directory, exist_ok=True)\n",
        "\n",
        "  # File path\n",
        "  file_path = os.path.join(directory, name_of_batch_validation_details_csv + f\"{epoch}.csv\")\n",
        "\n",
        "  # Write to CSV file\n",
        "  with open(file_path, \"w\", newline=\"\") as file:\n",
        "      writer = csv.writer(file)\n",
        "      # Write header\n",
        "      writer.writerow([\"Batch Count Validation\", \"Batch Validation Loss\"])\n",
        "      # Write data rows\n",
        "      for batch_count, batch_val_score in zip(batch_count_val, batch_val_score_values):\n",
        "          writer.writerow([batch_count, batch_val_score])\n",
        "\n",
        "  print(f\"CSV file for validation, epoch-{epoch}has been created at: {file_path}\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {
        "id": "ijkQ3VlU-unm"
      },
      "outputs": [],
      "source": [
        "# Function to save the data dynamically\n",
        "def save_progress(epoch_count, loss_values, val_loss_values, filepath= Path(save_progress_path)):\n",
        "    # Create a dictionary of lists\n",
        "    data = {\n",
        "        'epoch': epoch_count,\n",
        "        'train_loss': loss_values,\n",
        "        'val_loss': [None] * (len(epoch_count) - len(val_loss_values)) + val_loss_values\n",
        "    }\n",
        "\n",
        "    # Create a DataFrame\n",
        "    df = pd.DataFrame(data)\n",
        "\n",
        "    # Save DataFrame to CSV (mode='w' to write from scratch each time, mode='a' to append)\n",
        "    df.to_csv(filepath, index=False, mode='w')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {
        "id": "PTogqmQgWkZC"
      },
      "outputs": [],
      "source": [
        "# Function to save the data dynamically\n",
        "def save_dice_score(epoch_count, val_score_values, filepath= Path(save_dice_score_path)):\n",
        "    # Create a dictionary of lists\n",
        "    data = {\n",
        "        'epoch': epoch_count,\n",
        "        'val_dice_scores': [None] * (len(epoch_count) - len(val_score_values)) + val_score_values\n",
        "    }\n",
        "\n",
        "    # Create a DataFrame\n",
        "    df = pd.DataFrame(data)\n",
        "\n",
        "    # Save DataFrame to CSV (mode='w' to write from scratch each time, mode='a' to append)\n",
        "    df.to_csv(filepath, index=False, mode='w')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {
        "id": "ntFd7ioHXHQv"
      },
      "outputs": [],
      "source": [
        "# Function to save the data dynamically\n",
        "def save_losses(bce_losses=bce_losses, dice_losses=dice_losses, focal_losses=focal_losses, filepath= Path(save_3losses_path)):\n",
        "    # Create a dictionary of lists\n",
        "    data = {\n",
        "        'bce': bce_losses,\n",
        "        'dice': dice_losses,\n",
        "        'focal': focal_losses\n",
        "    }\n",
        "\n",
        "    # Create a DataFrame\n",
        "    df = pd.DataFrame(data)\n",
        "\n",
        "    # Save DataFrame to CSV (mode='w' to write from scratch each time, mode='a' to append)\n",
        "    df.to_csv(filepath, index=False, mode='w')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WMFmkSOtXHQv"
      },
      "source": [
        "#### Training Pipeline"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0bPiRNBuK2S6",
        "outputId": "de7bef60-e147-4263-f2f6-374254bd45cc",
        "scrolled": false
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [15:51<00:00,  1.60s/it, learning_rate=1e-5, loss=0.922]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-1has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_1.csv\n",
            "EPOCH: 1, TRAIN LOSS: 0.9221551354191885\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.77830 on (0.75, 250.0, 0.3): 100%|██████████| 268/268 [06:43<00:00,  1.51s/it, dice_score=0.784, val_loss=1.1]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-1has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics1.csv\n",
            "EPOCH: 1, TRAIN LOSS: 0.9221551354191885, VAL DICE: 0.7841194347758371\n",
            "Validation score improved (-inf --> 0.7841194347758371). Model saved at /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/m_1024_CHECKPOINTS_epoch1_bst_model1024_fold4_0.7841.tar!\n",
            "Checkpoint saved on epoch - 1 with dice score - 0.7841194347758371\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [05:58<00:00,  1.66it/s, learning_rate=1e-5, loss=0.919]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-2has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_2.csv\n",
            "EPOCH: 2, TRAIN LOSS: 0.9192208712842284\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.77685 on (0.6, 3000, 0.3): 100%|██████████| 268/268 [00:54<00:00,  4.89it/s, dice_score=0.78, val_loss=1.05]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-2has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics2.csv\n",
            "EPOCH: 2, TRAIN LOSS: 0.9192208712842284, VAL DICE: 0.7799081636536486\n",
            "EarlyStopping counter: 1 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:44<00:00,  2.09it/s, learning_rate=5.05e-6, loss=0.912]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-3has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_3.csv\n",
            "EPOCH: 3, TRAIN LOSS: 0.9118364689730797\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.81557 on (0.6, 3000, 0.3): 100%|██████████| 268/268 [00:55<00:00,  4.85it/s, dice_score=0.82, val_loss=1.02]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-3has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics3.csv\n",
            "EPOCH: 3, TRAIN LOSS: 0.9118364689730797, VAL DICE: 0.8196133856152807\n",
            "Validation score improved (0.7841194347758371 --> 0.8196133856152807). Model saved at /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/m_1024_CHECKPOINTS_epoch3_bst_model1024_fold4_0.8196.tar!\n",
            "Checkpoint saved on epoch - 3 with dice score - 0.8196133856152807\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:05<00:00,  2.42it/s, learning_rate=1e-5, loss=0.917]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-4has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_4.csv\n",
            "EPOCH: 4, TRAIN LOSS: 0.9165379207675196\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.50367 on (0.75, 2000, 0.4): 100%|██████████| 268/268 [00:55<00:00,  4.80it/s, dice_score=0.502, val_loss=1.25]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-4has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics4.csv\n",
            "EPOCH: 4, TRAIN LOSS: 0.9165379207675196, VAL DICE: 0.5021134955617592\n",
            "EarlyStopping counter: 1 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:03<00:00,  2.44it/s, learning_rate=8.55e-6, loss=0.912]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-5has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_5.csv\n",
            "EPOCH: 5, TRAIN LOSS: 0.9122614010041501\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.19289 on (0.75, 2000, 0.4): 100%|██████████| 268/268 [00:56<00:00,  4.77it/s, dice_score=0.192, val_loss=1.79]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-5has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics5.csv\n",
            "EPOCH: 5, TRAIN LOSS: 0.9122614010041501, VAL DICE: 0.19152933813368817\n",
            "EarlyStopping counter: 2 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:04<00:00,  2.44it/s, learning_rate=5.05e-6, loss=0.91]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-6has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_6.csv\n",
            "EPOCH: 6, TRAIN LOSS: 0.9097772496087211\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.77830 on (0.6, 500.0, 0.35): 100%|██████████| 268/268 [00:56<00:00,  4.73it/s, dice_score=0.784, val_loss=1.13]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-6has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics6.csv\n",
            "EPOCH: 6, TRAIN LOSS: 0.9097772496087211, VAL DICE: 0.7837868456017937\n",
            "EarlyStopping counter: 3 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:04<00:00,  2.43it/s, learning_rate=1.55e-6, loss=0.906]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-7has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_7.csv\n",
            "EPOCH: 7, TRAIN LOSS: 0.9055663200987487\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.77889 on (0.6, 3000, 0.3): 100%|██████████| 268/268 [00:56<00:00,  4.71it/s, dice_score=0.794, val_loss=0.971]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-7has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics7.csv\n",
            "EPOCH: 7, TRAIN LOSS: 0.9055663200987487, VAL DICE: 0.7935205935496978\n",
            "EarlyStopping counter: 4 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:04<00:00,  2.43it/s, learning_rate=1e-5, loss=0.909]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-8has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_8.csv\n",
            "EPOCH: 8, TRAIN LOSS: 0.9089200899380596\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.76195 on (0.75, 2000, 0.3): 100%|██████████| 268/268 [00:57<00:00,  4.68it/s, dice_score=0.769, val_loss=1.05]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-8has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics8.csv\n",
            "EPOCH: 8, TRAIN LOSS: 0.9089200899380596, VAL DICE: 0.7688817533732455\n",
            "EarlyStopping counter: 5 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:03<00:00,  2.44it/s, learning_rate=9.62e-6, loss=0.914]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-9has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_9.csv\n",
            "EPOCH: 9, TRAIN LOSS: 0.9140807459334366\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.25344 on (0.75, 2000, 0.4): 100%|██████████| 268/268 [00:57<00:00,  4.68it/s, dice_score=0.247, val_loss=1.44]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-9has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics9.csv\n",
            "EPOCH: 9, TRAIN LOSS: 0.9140807459334366, VAL DICE: 0.2473350922394283\n",
            "EarlyStopping counter: 6 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:06<00:00,  2.41it/s, learning_rate=8.55e-6, loss=0.905]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-10has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_10.csv\n",
            "EPOCH: 10, TRAIN LOSS: 0.905291808252575\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.77811 on (0.75, 2000, 0.4): 100%|██████████| 268/268 [00:57<00:00,  4.63it/s, dice_score=0.782, val_loss=1.19]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-10has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics10.csv\n",
            "EPOCH: 10, TRAIN LOSS: 0.905291808252575, VAL DICE: 0.7822311712467849\n",
            "EarlyStopping counter: 7 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:04<00:00,  2.44it/s, learning_rate=6.94e-6, loss=0.898]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-11has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_11.csv\n",
            "EPOCH: 11, TRAIN LOSS: 0.8977756171667275\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.81117 on (0.6, 3000, 0.3): 100%|██████████| 268/268 [00:58<00:00,  4.60it/s, dice_score=0.812, val_loss=1.03]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-11has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics11.csv\n",
            "EPOCH: 11, TRAIN LOSS: 0.8977756171667275, VAL DICE: 0.8121446715237434\n",
            "EarlyStopping counter: 8 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:03<00:00,  2.44it/s, learning_rate=5.05e-6, loss=0.897]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-12has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_12.csv\n",
            "EPOCH: 12, TRAIN LOSS: 0.8966217079082457\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.77951 on (0.6, 750.0, 0.3): 100%|██████████| 268/268 [00:58<00:00,  4.59it/s, dice_score=0.785, val_loss=1.09]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-12has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics12.csv\n",
            "EPOCH: 12, TRAIN LOSS: 0.8966217079082457, VAL DICE: 0.7847783422414186\n",
            "EarlyStopping counter: 9 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:03<00:00,  2.44it/s, learning_rate=3.16e-6, loss=0.898]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-13has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_13.csv\n",
            "EPOCH: 13, TRAIN LOSS: 0.8983870119607749\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.78266 on (0.75, 2000, 0.3): 100%|██████████| 268/268 [00:58<00:00,  4.56it/s, dice_score=0.792, val_loss=1.02]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-13has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics13.csv\n",
            "EPOCH: 13, TRAIN LOSS: 0.8983870119607749, VAL DICE: 0.7915849298980259\n",
            "EarlyStopping counter: 10 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:03<00:00,  2.44it/s, learning_rate=1.55e-6, loss=0.894]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-14has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_14.csv\n",
            "EPOCH: 14, TRAIN LOSS: 0.8941678300625135\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.82074 on (0.6, 3000, 0.4): 100%|██████████| 268/268 [00:59<00:00,  4.53it/s, dice_score=0.821, val_loss=0.948]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-14has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics14.csv\n",
            "EPOCH: 14, TRAIN LOSS: 0.8941678300625135, VAL DICE: 0.8206615664062022\n",
            "Validation score improved (0.8196133856152807 --> 0.8206615664062022). Model saved at /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/m_1024_CHECKPOINTS_epoch14_bst_model1024_fold4_0.8207.tar!\n",
            "Checkpoint saved on epoch - 14 with dice score - 0.8206615664062022\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:04<00:00,  2.44it/s, learning_rate=4.77e-7, loss=0.896]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-15has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_15.csv\n",
            "EPOCH: 15, TRAIN LOSS: 0.8958942259059233\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.83006 on (0.6, 3000, 0.3): 100%|██████████| 268/268 [00:59<00:00,  4.54it/s, dice_score=0.829, val_loss=0.945]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-15has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics15.csv\n",
            "EPOCH: 15, TRAIN LOSS: 0.8958942259059233, VAL DICE: 0.8294179083124175\n",
            "Validation score improved (0.8206615664062022 --> 0.8294179083124175). Model saved at /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/m_1024_CHECKPOINTS_epoch15_bst_model1024_fold4_0.8294.tar!\n",
            "Checkpoint saved on epoch - 15 with dice score - 0.8294179083124175\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:03<00:00,  2.44it/s, learning_rate=1e-5, loss=0.905]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-16has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_16.csv\n",
            "EPOCH: 16, TRAIN LOSS: 0.9046423892013166\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.81302 on (0.6, 2000, 0.3): 100%|██████████| 268/268 [00:59<00:00,  4.48it/s, dice_score=0.817, val_loss=1.01]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-16has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics16.csv\n",
            "EPOCH: 16, TRAIN LOSS: 0.9046423892013166, VAL DICE: 0.8168905017831881\n",
            "EarlyStopping counter: 1 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:03<00:00,  2.44it/s, learning_rate=9.9e-6, loss=0.904]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-17has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_17.csv\n",
            "EPOCH: 17, TRAIN LOSS: 0.9040544192330177\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.76234 on (0.75, 2000, 0.3): 100%|██████████| 268/268 [01:00<00:00,  4.46it/s, dice_score=0.766, val_loss=1.15]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-17has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics17.csv\n",
            "EPOCH: 17, TRAIN LOSS: 0.9040544192330177, VAL DICE: 0.7656490185054221\n",
            "EarlyStopping counter: 2 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:03<00:00,  2.44it/s, learning_rate=9.62e-6, loss=0.893]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-18has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_18.csv\n",
            "EPOCH: 18, TRAIN LOSS: 0.8926398721061835\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.75610 on (0.6, 3000, 0.3): 100%|██████████| 268/268 [01:00<00:00,  4.44it/s, dice_score=0.771, val_loss=1.05]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-18has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics18.csv\n",
            "EPOCH: 18, TRAIN LOSS: 0.8926398721061835, VAL DICE: 0.7705736890750975\n",
            "EarlyStopping counter: 3 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:04<00:00,  2.44it/s, learning_rate=9.17e-6, loss=0.901]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-19has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_19.csv\n",
            "EPOCH: 19, TRAIN LOSS: 0.900634253726286\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.56522 on (0.75, 2000, 0.4): 100%|██████████| 268/268 [01:00<00:00,  4.42it/s, dice_score=0.568, val_loss=1.17]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-19has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics19.csv\n",
            "EPOCH: 19, TRAIN LOSS: 0.900634253726286, VAL DICE: 0.5677835713780254\n",
            "EarlyStopping counter: 4 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:04<00:00,  2.44it/s, learning_rate=8.55e-6, loss=0.898]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-20has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_20.csv\n",
            "EPOCH: 20, TRAIN LOSS: 0.8978940673235084\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.77830 on (0.75, 250.0, 0.3): 100%|██████████| 268/268 [01:01<00:00,  4.38it/s, dice_score=0.784, val_loss=1.09]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-20has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics20.csv\n",
            "EPOCH: 20, TRAIN LOSS: 0.8978940673235084, VAL DICE: 0.7837868456017937\n",
            "EarlyStopping counter: 5 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:04<00:00,  2.43it/s, learning_rate=7.8e-6, loss=0.895]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-21has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_21.csv\n",
            "EPOCH: 21, TRAIN LOSS: 0.8945616490700665\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.77067 on (0.75, 2000, 0.3): 100%|██████████| 268/268 [01:01<00:00,  4.34it/s, dice_score=0.778, val_loss=1.07]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-21has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics21.csv\n",
            "EPOCH: 21, TRAIN LOSS: 0.8945616490700665, VAL DICE: 0.7777885572331759\n",
            "EarlyStopping counter: 6 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:04<00:00,  2.44it/s, learning_rate=6.94e-6, loss=0.898]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-22has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_22.csv\n",
            "EPOCH: 22, TRAIN LOSS: 0.8975558381120697\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.81506 on (0.75, 2000, 0.3): 100%|██████████| 268/268 [01:01<00:00,  4.33it/s, dice_score=0.821, val_loss=0.977]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-22has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics22.csv\n",
            "EPOCH: 22, TRAIN LOSS: 0.8975558381120697, VAL DICE: 0.8209269681840863\n",
            "EarlyStopping counter: 7 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:04<00:00,  2.44it/s, learning_rate=6.02e-6, loss=0.887]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-23has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_23.csv\n",
            "EPOCH: 23, TRAIN LOSS: 0.8867435239944137\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.76116 on (0.6, 3000, 0.3): 100%|██████████| 268/268 [01:02<00:00,  4.31it/s, dice_score=0.774, val_loss=1.05]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-23has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics23.csv\n",
            "EPOCH: 23, TRAIN LOSS: 0.8867435239944137, VAL DICE: 0.7740473896917552\n",
            "EarlyStopping counter: 8 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:04<00:00,  2.43it/s, learning_rate=5.05e-6, loss=0.894]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-24has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_24.csv\n",
            "EPOCH: 24, TRAIN LOSS: 0.8944177654610962\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.76208 on (0.75, 2000, 0.3): 100%|██████████| 268/268 [01:02<00:00,  4.28it/s, dice_score=0.774, val_loss=1.04]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-24has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics24.csv\n",
            "EPOCH: 24, TRAIN LOSS: 0.8944177654610962, VAL DICE: 0.7743444413807351\n",
            "EarlyStopping counter: 9 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:04<00:00,  2.43it/s, learning_rate=4.08e-6, loss=0.885]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-25has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_25.csv\n",
            "EPOCH: 25, TRAIN LOSS: 0.8854661204233891\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.77028 on (0.75, 2000, 0.3): 100%|██████████| 268/268 [01:02<00:00,  4.29it/s, dice_score=0.78, val_loss=1.04]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-25has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics25.csv\n",
            "EPOCH: 25, TRAIN LOSS: 0.8854661204233891, VAL DICE: 0.7795697323329565\n",
            "EarlyStopping counter: 10 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:04<00:00,  2.44it/s, learning_rate=3.16e-6, loss=0.876]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-26has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_26.csv\n",
            "EPOCH: 26, TRAIN LOSS: 0.8763040107839247\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.81631 on (0.6, 3000, 0.3): 100%|██████████| 268/268 [01:02<00:00,  4.27it/s, dice_score=0.815, val_loss=1.02]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-26has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics26.csv\n",
            "EPOCH: 26, TRAIN LOSS: 0.8763040107839247, VAL DICE: 0.8152638571975142\n",
            "EarlyStopping counter: 11 out of 12\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 595/595 [04:04<00:00,  2.43it/s, learning_rate=2.3e-6, loss=0.887]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for epoch-27has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_train_m1024_b8/Model1024_HPT_TRAIN_1024_b_8_epoch_27.csv\n",
            "EPOCH: 27, TRAIN LOSS: 0.8867978241263318\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "score: 0.82476 on (0.6, 3000, 0.3): 100%|██████████| 268/268 [01:03<00:00,  4.21it/s, dice_score=0.825, val_loss=0.948]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV file for validation, epoch-27has been created at: /content/drive/MyDrive/Saved Models/model1024/Model1024_HPT_EXP_ATT_4_BS_8_PHASE_3_step_1_explore/Model1024_HPT_vali_m1024_b8/Model1024_HPT_1024_b_8_Validation_Metrics27.csv\n",
            "EPOCH: 27, TRAIN LOSS: 0.8867978241263318, VAL DICE: 0.8247871555851205\n",
            "EarlyStopping counter: 12 out of 12\n",
            "\n",
            "\n",
            " -------------- EARLY STOPPING -------------- \n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "till_epoch = EPOCHS\n",
        "model.to(DEVICE)\n",
        "model.train()\n",
        "# Lists to store epoch-to-epoch progress details\n",
        "epoch_count = []\n",
        "loss_values = []\n",
        "val_score_values = []\n",
        "val_loss_values = []\n",
        "\n",
        "# Training Loop\n",
        "for epoch in range(1, till_epoch+1):\n",
        "    epoch_count.append(epoch)\n",
        "    loss, batch_count_train, batch_train_loss = train_one_epoch(train_dataloader, model, optimizer, criterion)\n",
        "    loss_values.append(loss)\n",
        "    store_batch_training_details(epoch, batch_count_train, batch_train_loss)\n",
        "    if not isinstance(scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau):\n",
        "        scheduler.step()\n",
        "    # Storing Training details for plotting curves and inference\n",
        "    print(f\"EPOCH: {epoch}, TRAIN LOSS: {loss}\")\n",
        "\n",
        "    \"\"\"-------------------VALIDATION---------------------\"\"\"\n",
        "    dice, batch_count_val, batch_val_score_values, val_loss = evaluate(val_dataloader, model, epoch=epoch) # Evaluates model performance by calculating the dice coefficient\n",
        "    val_score_values.append(dice)\n",
        "    val_loss_values.append(val_loss)\n",
        "    # Storing Validation details for plotting curves and inference\n",
        "    store_batch_validation_details(epoch, batch_count_val, batch_val_score_values)\n",
        "    print(f\"EPOCH: {epoch}, TRAIN LOSS: {loss}, VAL DICE: {dice}\")\n",
        "\n",
        "    save_progress(epoch_count, loss_values, val_loss_values)\n",
        "    save_dice_score(epoch_count, val_score_values)\n",
        "    save_losses()\n",
        "# \"\"\"\" If it is necessary to save model's state dict as checkpoint, do the essential changes\n",
        "#      in Early Stopping class.\"\"\"\n",
        "\n",
        "    if isinstance(scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau):\n",
        "        scheduler.step(dice)\n",
        "\n",
        "    es(epoch, dice, model, optimizer, loss, model_path= model_checkpoint_path + f\"_epoch{epoch}_bst_model{IMG_SIZE}_fold{FOLD_ID}_{np.round(dice,4)}.tar\")\n",
        "    if es.early_stop:\n",
        "        print('\\n\\n -------------- EARLY STOPPING -------------- \\n\\n')\n",
        "        break"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gJFz9wAhzL4k"
      },
      "source": [
        "# Training Dynamics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 607
        },
        "id": "n2BXTyFrPxzy",
        "outputId": "beeb869f-17b3-43d1-9d36-6cd1adc7eea4"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA90AAAJOCAYAAACqS2TfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACoFklEQVR4nOzdd3xT1f/H8Xc6KLSl7F1kyVRAVFRUlixBUaw4EBUUHF8n4l4IiPpz454IooIKAupXZG/1q6LiVkT2XkJpoSu9vz+Ot4OutE1yb9LX8/HoIzfJTXLa3qZ533PO53gsy7IEAAAAAAD8LsLpBgAAAAAAEK4I3QAAAAAABAihGwAAAACAACF0AwAAAAAQIIRuAAAAAAAChNANAAAAAECAELoBAAAAAAgQQjcAAAAAAAFC6AYAAAAAIEAI3QCAkDN8+HA1bdq0TI8dO3asPB6PfxvkMhs3bpTH49GUKVOC/toej0djx47NuT5lyhR5PB5t3LixxMc2bdpUw4cP92t7ynOsAADgD4RuAIDfeDwen76WLVvmdFMrvFtuuUUej0fr1q0rcp/7779fHo9HP/30UxBbVnrbt2/X2LFjtWbNGqebksM+8fHUU0853RQAgMOinG4AACB8vPPOO/muT506VQsXLixwe9u2bcv1Om+88Yays7PL9NgHHnhA99xzT7lePxwMHTpUL7zwgqZNm6YxY8YUus/06dPVvn17dejQocyvc8UVV+jSSy9VTExMmZ+jJNu3b9e4cePUtGlTnXDCCfnuK8+xAgCAPxC6AQB+c/nll+e7/r///U8LFy4scPvRDh8+rNjYWJ9fJzo6ukztk6SoqChFRfHv79RTT9Wxxx6r6dOnFxq6v/rqK23YsEH/93//V67XiYyMVGRkZLmeozzKc6wAAOAPDC8HAARVjx49dPzxx+u7775Tt27dFBsbq/vuu0+S9PHHH+ucc85Rw4YNFRMToxYtWujhhx+W1+vN9xxHz9PNO5T39ddfV4sWLRQTE6POnTvr22+/zffYwuZ0ezwe3XTTTZozZ46OP/54xcTE6LjjjtO8efMKtH/ZsmU6+eSTVblyZbVo0UKvvfaaz/PEV65cqYsuukjHHHOMYmJi1LhxY9122206cuRIge8vPj5e27Zt06BBgxQfH686derojjvuKPCzOHDggIYPH65q1aqpevXqGjZsmA4cOFBiWyTT2/3HH3/o+++/L3DftGnT5PF4NGTIEGVkZGjMmDE66aSTVK1aNcXFxalr165aunRpia9R2Jxuy7I0YcIEJSYmKjY2Vj179tSvv/5a4LH79+/XHXfcofbt2ys+Pl4JCQnq37+/fvzxx5x9li1bps6dO0uSrrrqqpwpDPZ89sLmdKempur2229X48aNFRMTo9atW+upp56SZVn59ivNcVFWu3fv1ogRI1SvXj1VrlxZHTt21Ntvv11gv/fff18nnXSSqlatqoSEBLVv317PPfdczv2ZmZkaN26cWrZsqcqVK6tWrVo688wztXDhQr+1FQBQNpzqBwAE3b59+9S/f39deumluvzyy1WvXj1JJqDFx8dr9OjRio+P15IlSzRmzBglJyfrySefLPF5p02bpkOHDum6666Tx+PRE088oaSkJK1fv77EHs9Vq1Zp1qxZuuGGG1S1alU9//zzuvDCC7V582bVqlVLkvTDDz/o7LPPVoMGDTRu3Dh5vV6NHz9ederU8en7njFjhg4fPqz//Oc/qlWrlr755hu98MIL2rp1q2bMmJFvX6/Xq379+unUU0/VU089pUWLFunpp59WixYt9J///EeSCa/nn3++Vq1apeuvv15t27bV7NmzNWzYMJ/aM3ToUI0bN07Tpk3TiSeemO+1P/zwQ3Xt2lXHHHOM9u7dqzfffFNDhgzRNddco0OHDmnSpEnq16+fvvnmmwJDuksyZswYTZgwQQMGDNCAAQP0/fffq2/fvsrIyMi33/r16zVnzhxddNFFatasmXbt2qXXXntN3bt312+//aaGDRuqbdu2Gj9+vMaMGaNrr71WXbt2lSSdfvrphb62ZVk677zztHTpUo0YMUInnHCC5s+frzvvvFPbtm3Ts88+m29/X46Lsjpy5Ih69OihdevW6aabblKzZs00Y8YMDR8+XAcOHNCtt94qSVq4cKGGDBmiXr166fHHH5ck/f777/riiy9y9hk7dqwee+wxjRw5UqeccoqSk5O1evVqff/99+rTp0+52gkAKCcLAIAAufHGG62j/9V0797dkmS9+uqrBfY/fPhwgduuu+46KzY21kpLS8u5bdiwYVaTJk1yrm/YsMGSZNWqVcvav39/zu0ff/yxJcn69NNPc2576KGHCrRJklWpUiVr3bp1Obf9+OOPliTrhRdeyLlt4MCBVmxsrLVt27ac2/766y8rKiqqwHMWprDv77HHHrM8Ho+1adOmfN+fJGv8+PH59u3UqZN10kkn5VyfM2eOJcl64okncm7LysqyunbtakmyJk+eXGKbOnfubCUmJlperzfntnnz5lmSrNdeey3nOdPT0/M97p9//rHq1atnXX311flul2Q99NBDOdcnT55sSbI2bNhgWZZl7d6926pUqZJ1zjnnWNnZ2Tn73XfffZYka9iwYTm3paWl5WuXZZnfdUxMTL6fzbffflvk93v0sWL/zCZMmJBvv8GDB1sejyffMeDrcVEY+5h88skni9xn4sSJliTr3XffzbktIyPD6tKlixUfH28lJydblmVZt956q5WQkGBlZWUV+VwdO3a0zjnnnGLbBABwBsPLAQBBFxMTo6uuuqrA7VWqVMnZPnTokPbu3auuXbvq8OHD+uOPP0p83ksuuUQ1atTIuW73eq5fv77Ex/bu3VstWrTIud6hQwclJCTkPNbr9WrRokUaNGiQGjZsmLPfscceq/79+5f4/FL+7y81NVV79+7V6aefLsuy9MMPPxTY//rrr893vWvXrvm+l7lz5yoqKiqn51syc6hvvvlmn9ojmXn4W7du1YoVK3JumzZtmipVqqSLLroo5zkrVaokScrOztb+/fuVlZWlk08+udCh6cVZtGiRMjIydPPNN+cbkj9q1KgC+8bExCgiwnxU8Xq92rdvn+Lj49W6detSv65t7ty5ioyM1C233JLv9ttvv12WZenzzz/Pd3tJx0V5zJ07V/Xr19eQIUNybouOjtYtt9yilJQULV++XJJUvXp1paamFjtUvHr16vr111/1119/lbtdAAD/InQDAIKuUaNGOSEur19//VUXXHCBqlWrpoSEBNWpUyenCNvBgwdLfN5jjjkm33U7gP/zzz+lfqz9ePuxu3fv1pEjR3TssccW2K+w2wqzefNmDR8+XDVr1syZp929e3dJBb+/ypUrFxi2nrc9krRp0yY1aNBA8fHx+fZr3bq1T+2RpEsvvVSRkZGaNm2aJCktLU2zZ89W//79853AePvtt9WhQ4ec+cJ16tTRZ5995tPvJa9NmzZJklq2bJnv9jp16uR7PckE/GeffVYtW7ZUTEyMateurTp16uinn34q9evmff2GDRuqatWq+W63K+rb7bOVdFyUx6ZNm9SyZcucEwtFteWGG25Qq1at1L9/fyUmJurqq68uMK98/PjxOnDggFq1aqX27dvrzjvvdP1SbwBQURC6AQBBl7fH13bgwAF1795dP/74o8aPH69PP/1UCxcuzJnD6suyT0VVybaOKpDl78f6wuv1qk+fPvrss8909913a86cOVq4cGFOwa+jv79gVfyuW7eu+vTpo48++kiZmZn69NNPdejQIQ0dOjRnn3fffVfDhw9XixYtNGnSJM2bN08LFy7UWWedFdDluB599FGNHj1a3bp107vvvqv58+dr4cKFOu6444K2DFigjwtf1K1bV2vWrNEnn3ySMx+9f//++ebud+vWTX///bfeeustHX/88XrzzTd14okn6s033wxaOwEAhaOQGgDAFZYtW6Z9+/Zp1qxZ6tatW87tGzZscLBVuerWravKlStr3bp1Be4r7Laj/fzzz1q7dq3efvttXXnllTm3l6e6dJMmTbR48WKlpKTk6+3+888/S/U8Q4cO1bx58/T5559r2rRpSkhI0MCBA3Punzlzppo3b65Zs2blGxL+0EMPlanNkvTXX3+pefPmObfv2bOnQO/xzJkz1bNnT02aNCnf7QcOHFDt2rVzrvtSOT7v6y9atEiHDh3K19ttT1+w2xcMTZo00U8//aTs7Ox8vd2FtaVSpUoaOHCgBg4cqOzsbN1www167bXX9OCDD+aMtKhZs6auuuoqXXXVVUpJSVG3bt00duxYjRw5MmjfEwCgIHq6AQCuYPco5u1BzMjI0Msvv+xUk/KJjIxU7969NWfOHG3fvj3n9nXr1hWYB1zU46X8359lWfmWfSqtAQMGKCsrS6+88krObV6vVy+88EKpnmfQoEGKjY3Vyy+/rM8//1xJSUmqXLlysW3/+uuv9dVXX5W6zb1791Z0dLReeOGFfM83ceLEAvtGRkYW6FGeMWOGtm3blu+2uLg4SfJpqbQBAwbI6/XqxRdfzHf7s88+K4/H4/P8fH8YMGCAdu7cqQ8++CDntqysLL3wwguKj4/PmXqwb9++fI+LiIhQhw4dJEnp6emF7hMfH69jjz02534AgHPo6QYAuMLpp5+uGjVqaNiwYbrlllvk8Xj0zjvvBHUYb0nGjh2rBQsW6IwzztB//vOfnPB2/PHHa82aNcU+tk2bNmrRooXuuOMObdu2TQkJCfroo4/KNTd44MCBOuOMM3TPPfdo48aNateunWbNmlXq+c7x8fEaNGhQzrzuvEPLJencc8/VrFmzdMEFF+icc87Rhg0b9Oqrr6pdu3ZKSUkp1WvZ640/9thjOvfcczVgwAD98MMP+vzzz/P1XtuvO378eF111VU6/fTT9fPPP+u9997L10MuSS1atFD16tX16quvqmrVqoqLi9Opp56qZs2aFXj9gQMHqmfPnrr//vu1ceNGdezYUQsWLNDHH3+sUaNG5Sua5g+LFy9WWlpagdsHDRqka6+9Vq+99pqGDx+u7777Tk2bNtXMmTP1xRdfaOLEiTk98SNHjtT+/ft11llnKTExUZs2bdILL7ygE044IWf+d7t27dSjRw+ddNJJqlmzplavXq2ZM2fqpptu8uv3AwAoPUI3AMAVatWqpf/+97+6/fbb9cADD6hGjRq6/PLL1atXL/Xr18/p5kmSTjrpJH3++ee644479OCDD6px48YaP368fv/99xKrq0dHR+vTTz/VLbfcoscee0yVK1fWBRdcoJtuukkdO3YsU3siIiL0ySefaNSoUXr33Xfl8Xh03nnn6emnn1anTp1K9VxDhw7VtGnT1KBBA5111ln57hs+fLh27typ1157TfPnz1e7du307rvvasaMGVq2bFmp2z1hwgRVrlxZr776qpYuXapTTz1VCxYs0DnnnJNvv/vuu0+pqamaNm2aPvjgA5144on67LPPdM899+TbLzo6Wm+//bbuvfdeXX/99crKytLkyZMLDd32z2zMmDH64IMPNHnyZDVt2lRPPvmkbr/99lJ/LyWZN29egaJnktS0aVMdf/zxWrZsme655x69/fbbSk5OVuvWrTV58mQNHz48Z9/LL79cr7/+ul5++WUdOHBA9evX1yWXXKKxY8fmDEu/5ZZb9Mknn2jBggVKT09XkyZNNGHCBN15551+/54AAKXjsdzUhQAAQAgaNGgQyzUBAIBCMacbAIBSOHLkSL7rf/31l+bOnasePXo40yAAAOBq9HQDAFAKDRo00PDhw9W8eXNt2rRJr7zyitLT0/XDDz8UWHsaAACAOd0AAJTC2WefrenTp2vnzp2KiYlRly5d9OijjxK4AQBAoejpBgAAAAAgQJjTDQAAAABAgBC6AQAAAAAIkAo3pzs7O1vbt29X1apV5fF4nG4OAAAAACAEWZalQ4cOqWHDhoqIKLo/u8KF7u3bt6tx48ZONwMAAAAAEAa2bNmixMTEIu+vcKG7atWqkswPJiEhocj9MjMztWDBAvXt21fR0dHBah6Qg2MQTuL4g9M4BuEkjj84jWMwNCQnJ6tx48Y5GbMoFS5020PKExISSgzdsbGxSkhI4ECHIzgG4SSOPziNYxBO4viD0zgGQ0tJ05YppAYAAAAAQIAQugEAAAAACBBCNwAAAAAAAVLh5nQDAAAACC/Z2dnKyMhwuhl+k5mZqaioKKWlpcnr9TrdnAorOjpakZGR5X4eQjcAAACAkJWRkaENGzYoOzvb6ab4jWVZql+/vrZs2VJikS4EVvXq1VW/fv1y/R4I3QAAAABCkmVZ2rFjhyIjI9W4cWNFRITH7Nns7GylpKQoPj4+bL6nUGNZlg4fPqzdu3dLkho0aFDm5yJ0AwAAAAhJWVlZOnz4sBo2bKjY2Finm+M39nD5ypUrE7odVKVKFUnS7t27Vbdu3TIPNec3CAAAACAk2fOdK1Wq5HBLEK7skzmZmZllfg5CNwAAAICQxrxnBIo/ji1CNwAAAAAAAULoBgAAAIAQ17RpU02cONHpZqAQhG4AAAAAFZrXKy1bJk2fbi4DuTS2x+Mp9mvs2LFlet5vv/1W1157bbna1qNHD40aNapcz4GCqF4OAAAAoMKaNUu69VZp69bc2xITpeeek5KS/P96O3bsyNn+4IMPNGbMGP355585t8XHx+dsW5alrKwsRUWVHNvq1Knj34bCbxzt6V6xYoUGDhyohg0byuPxaM6cOSU+5r333lPHjh0VGxurBg0a6Oqrr9a+ffsC31gAAAAAYWXWLGnw4PyBW5K2bTO3z5rl/9esX79+zle1atXk8Xhyrv/xxx+qWrWqPv/8c/Xo0UNVqlTRqlWr9Pfff+v8889XvXr1FB8fr86dO2vRokX5nvfo4eUej0dvvvmmLrjgAsXGxqply5b65JNPytX2jz76SMcdd5xiYmLUtGlTPf300/nuf/nll9WyZUtVrlxZ9erV0+DBg3Pumzlzptq3b68qVaqoVq1a6t27t1JTU8vVnlDhaOhOTU1Vx44d9dJLL/m0/xdffKErr7xSI0aM0K+//qoZM2bom2++0TXXXBPglgIAAABwO8uSUlN9+0pOlm65xTymsOeRTA94crJvz1fY85TVfffdp4ceeki//vqrOnTooJSUFA0YMECLFy/WDz/8oLPPPlsDBw7U5s2bi32ecePG6eKLL9ZPP/2kAQMGaOjQodq/f3+Z2vTdd9/p4osv1qWXXqqff/5ZY8eO1YMPPqgpU6ZIklavXq1bbrlF48eP159//ql58+apW7dukkzv/pAhQ3T11Vfr999/17Jly5SUlCTLnz80F3N0eHn//v3Vv39/n/f/6quv1LRpU91yyy2SpGbNmum6667T448/HqgmAs7weuVZvlyNVqyQJy5O6tlTiox0ulUAAACudviwlGd0drlYlukBr1bNt/1TUqS4OP+89tixY9WzZ08lJCQoIiJCNWvWVMeOHXPuf/jhhzV79mx98sknuummm4p8nuHDh2vIkCGSpEcffVTPP/+8vvnmG5199tmlbtMzzzyjXr166cEHH5QktWrVSr/99puefPJJDR8+XJs3b1ZcXJzOPfdcVa1aVU2aNFGnTp0kmdCdlZWlpKQkNWnSRJLUvn37UrchVIXUnO4uXbrovvvu09y5c9W/f3/t3r1bM2fO1IABA4p8THp6utLT03OuJycnSzKLmxe3wLl9X3kWQQfKwjN7tiJHj1bUtm06WZKeeUZWo0byPvOMrAsucLp5qCB4D4TTOAbhJI6/0JGZmSnLspSdnf3vl+TUYN7c1y/dYwq7POmkkyQp53tLSUnRuHHjNHfu3JwAe+TIEW3atCnnMXn3tx1//PE516tUqaKEhATt3Lkz3z5HO/o5bL///rvOO++8fPd16dJFEydOVGZmpnr16qUmTZqoefPm6tevn/r165cztL19+/bq1auX2rdvr759+6pPnz4aPHiwatSoUbofmAOys7NlWZYyMzMVeVQnmK/vESEVus844wy99957uuSSS5SWlqasrCwNHDiw2OHpjz32mMaNG1fg9gULFig2NrbE11y4cGG52gyURoOvvlLnwkZubNumyEsu0bd3360dXboEv2GosHgPhNM4BuEkjj/3i4qKUv369ZWSkqKMjIyc3mlffPlllC6+uORu8Q8/TNHpp2eVuF9WlhmKXhppaWmyLCunY/Dw4cOSlDPs+tChQ5Kk2267TcuWLdPDDz+sZs2aqUqVKho2bJhSUlJyHpudna20tLSc66ZNWfmu269x9G1598/IyCj0fq/Xq/T09Hz3HTlyRJLp2IyMjNSSJUu0atUqLVmyRGPGjNHYsWO1ZMkSVatWTTNmzNDXX3+tpUuX6vnnn9cDDzygRYsW5fR8u1VGRoaOHDmiFStWKCsr/3Fg/75KElKh+7ffftOtt96qMWPGqF+/ftqxY4fuvPNOXX/99Zo0aVKhj7n33ns1evTonOvJyclq3Lix+vbtq4SEhCJfKzMzUwsXLlSfPn0UHR3t9+8FKMDrVdSNN0qSPEfd5ZFkeTzq/N57yho7lqHmCDjeA+E0jkE4ieMvdKSlpWnLli2Kj49X5cqVJfk+HHzQICkx0dK2bZJlHf3pS/J4LCUmSoMGxQbso1flypXl8XhycondKWhXMK9atao8Ho9Wr16tq666SpdddpkkKSUlRVu2bFGlSpVyHhsREaHKlSvnyzh273bu9+QpsE9eUVFR+Z4zr+OOO06rV6/Od98PP/ygVq1a5euxPu+883TeeefpkUceUc2aNfXtt98q6d8y8H379lXfvn01YcIENWvWTIsWLdJtt91W+h9cEKWlpalKlSrq1q1bzjFmK+rkxdFCKnQ/9thjOuOMM3TnnXdKkjp06KC4uDh17dpVEyZMUIMGDQo8JiYmRjExMQVuj46O9ulN1Nf9gHL74gtTKrMInn9P3Ub/739Sjx7BaxcqNN4D4TSOQTiJ48/9vF6vPB6PIiIiFBFRumHlERFmWbDBgyWPJ38hNI9HkjyaOFGKji4YyP3FbvPRlx7TgJzvrWXLlpo9e7bOO+88eTwePfjgg8rOzs65P7fd+a8X9nMp6We1d+9e/fTTT/lua9Cgge644w517txZjzzyiC655BJ99dVXeumll/Tyyy8rIiJC//3vf7V+/Xp169ZNNWrU0Ny5c5Wdna22bdvq22+/1eLFi9W3b1/VrVtXX3/9tfbs2aN27dqV+vcWbBEREfJ4PIW+H/j6/uDu7/Aohw8fLvBLscfVV5TKdwhjedZs9Mt+AAAAKFZSkjRzptSoUf7bExPN7YFYp7ssnnnmGdWoUUOnn366Bg4cqH79+unEE08MyGtNmzZNnTp1yvf1xhtv6MQTT9SHH36o999/X8cff7zGjBmj8ePHa/jw4ZKk6tWra9asWTrrrLPUtm1bvfrqq5o+fbqOO+44JSQkaMWKFRowYIBatWqlBx54QE8//XSpimqHMkd7ulNSUrRu3bqc6xs2bNCaNWtUs2ZNHXPMMbr33nu1bds2TZ06VZI0cOBAXXPNNXrllVdyhpePGjVKp5xyiho2bOjUtwH4RyEjNcq1HwAAAEqUlCSdf760cqXp22jQQOraNTiz+YYPH54TWiWpR48eOYXM8g5dbtq0qZYsWZLvsTf+Oy3RtnHjxnzXC+uUPHDgQLHtWbZsWbH3X3jhhbrwwgsLve/MM88s8vFt27bVvHnzin3ucOZo6F69erV69uyZc92eez1s2DBNmTJFO3bsyLf23PDhw3Xo0CG9+OKLuv3221W9enWdddZZLBmG8NC1qzmtaiYWFbzf4zH3d+0a/LYBAACEschIZu8hcBwN3faZnKLYC63ndfPNN+vmm28OYKsAh0RG5k4sOtq/83o0cSJF1AAAAIAQElJzuoGwZ08sOrqghNsmFgEAAADwSUhVLwcqhH79pOzsnKtZr7yiqGuuoYcbAAAACEH0dANus2VL/uuNGhG4AQAAgBBF6AbcZtOm/Nd37nSmHQAAAADKjdANuE2eiv2S5GFdbgAAACBkEboBtzkqdNPTDQAAAIQuQjfgNv8OL7cSEyXR0w0AAACEMkI34Db/9nRbnTub6/R0AwAA4Cg9evTQqFGjcq43bdpUEydOLPYxHo9Hc+bMKfdr++t5KgpCN+A2dug+5RRJkofQDQAAEFher7RsmTR9urn0egP2UgMHDtTZZ59d6H0rV66Ux+PRTz/9VOrn/fbbb3XttdeWt3n5jB07VieccEKB23fs2KH+/fv79bWONmXKFFWvXj2grxEshG7ATbzenCXDrFNPNbft2CFZloONAgAACGOzZklNm0o9e0qXXWYumzY1twfAiBEjtHDhQm3durXAfZMnT9bJJ5+sDh06lPp569Spo9jYWH80sUT169dXTExMUF4rHBC6ATfZuVPKypIiI2V16iRJ8qSnSwcPOtwwAACAMDRrljR4sHR0AN62zdwegOB97rnnqk6dOpoyZUq+21NSUjRjxgyNGDFC+/bt04gRI9S4cWPFxsaqffv2mj59erHPe/Tw8r/++kvdunVT5cqV1a5dOy1cuLDAY+6++261atVKsbGxat68uR588EFlZmZKMj3N48aN048//iiPxyOPx5PT5qOHl//8888666yzVKVKFdWqVUvXXnutUlJScu4fPny4Bg0apKeeekoNGjRQrVq1dOONN+a8Vlls3rxZ559/vuLj45WQkKCLL75Yu3btyrn/xx9/VM+ePVW1alUlJCTopJNO0urVqyVJmzZt0sCBA1WjRg3FxcXpuOOO09y5c8vclpJEBeyZAZSeXbk8MVGKi1NGXJwqpaaa3u4wGV4DAAAQMJYlHT7s275er3TLLYWPKLQsyeORbr1V6t1biows+fliY81jShAVFaUrr7xSU6ZM0f333y/Pv4+ZMWOGvF6vhgwZouTkZJ1wwgm6//77Vb16dX322We64oor1KJFC53y7xTE4mRnZyspKUn16tXT119/rYMHD+ab/22rWrWqpkyZooYNG+rnn3/WNddco6pVq+quu+7SJZdcol9++UXz5s3TokWLJEnVqlUr8Bypqanq16+funTpom+//Va7d+/WyJEjddNNN+U7sbB06VI1aNBAS5cu1bp163TJJZfohBNO0DXXXFPi91PY92cH7uXLlysrK0s33nijLrnkEi1btkySNHToUHXq1EmvvPKKIiMjtWbNGkVHR0uSbrzxRmVkZGjFihWKi4vTb7/9pvj4+FK3w1eEbsBN/q1crmOOkSSl16iRG7rbtnWwYQAAACHg8GHJX+HJskwPeCFBs1ApKVJcnE+7Xn311XryySe1fPly9ejRQ5IZWn7hhReqWrVqqlq1qm6++WYlJCQoIiJCN998s+bPn68PP/zQp9C9aNEi/fHHH5o/f74aNmwoSXr00UcLzMN+4IEHcrabNm2qO+64Q++//77uuusuValSRfHx8YqKilL9+vWLfK1p06YpLS1NU6dOVdy/3/+LL76ogQMH6vHHH1e9evUkSTVq1NCLL76oyMhItWnTRuecc44WL15cptC9ePFi/fzzz9qwYYMaN24sSZo6daqOO+44ffvtt+rcubM2b96sO++8U23atJEktWzZMufxmzdv1oUXXqj27dtLkpo3b17qNpQGw8sBN7F7uv8N3Wk1apjrLBsGAAAQNtq0aaPTTz9db731liRp3bp1WrlypUaMGCFJ8nq9evLJJ9WxY0fVrFlT8fHxmj9/vjbbnxVL8Pvvv6tx48Y5gVuSunTpUmC/Dz74QGeccYbq16+v+Ph4PfDAAz6/Rt7X6tixY07glqQzzjhD2dnZ+vPPP3NuO+644xSZZ8RAgwYNtHv37lK9Vt7XbNy4cU7glqR27dqpevXq+v333yVJo0eP1siRI9W7d2/93//9n/7++++cfW+55RZNmDBBZ5xxhh566KEyFa4rDUI34Cb2m1yTJpKktJo1zXVCNwAAQMliY02Psy9fvs7hnTvXt+crZRGzESNG6KOPPtKhQ4c0efJktWjRQt27d5ckPfXUU3r11Vd15513aunSpVqzZo369eunjIyM0v5EivTVV19p6NChGjBggP773//qhx9+0P333+/X18jLHtpt83g8ys7ODshrSaby+q+//qpzzjlHS5YsUbt27TR79mxJ0siRI7V+/XpdccUV+vnnn3XyySfrhRdeCFhbCN2Am9DTDQAAUHYejxni7ctX376mjk5R87A9HqlxY7OfL8/nw3zuvC6++GJFRERo2rRpmjp1qq6++uqc+d1ffPGFBgwYoMsvv1wdO3ZU8+bNtXbtWp+fu23bttqyZYt25PkM+b///S/fPl9++aWaNGmi+++/XyeffLJatmypTfZUx39VqlRJ3hKWT2vbtq1+/PFHpaam5tz2xRdfKCIiQq1bt/a5zaVhf39b/l31R5J+++03HThwQO3atcu5rVWrVrrtttu0YMECJSUlafLkyTn3NW7cWNdff71mzZql22+/XW+88UZA2ioRugF3OXpOt108jdANAADgX5GR0nPPme2jA7N9feJE34qolUF8fLwuueQS3XvvvdqxY4eGDx+ec1/Lli21dOlSffnll/r999913XXX5avMXZLevXurVatWGjZsmH788UetXLlS999/f759WrZsqc2bN+v999/X33//reeffz6nJ9jWtGlTbdiwQWvWrNHevXuVnp5e4LWGDh2qypUra9iwYfrll1+0dOlS3Xzzzbriiity5nOXldfr1Zo1a/J9/f777+rdu7fat2+voUOH6vvvv9c333yjK6+8Ut27d9fJJ5+sI0eO6KabbtKyZcu0adMmffHFF/r222/V9t8aSaNGjdL8+fO1YcMGff/991q6dGnOfYFA6Abc5Ojh5fR0AwAABE5SkjRzptSoUf7bExPN7UlJAX35ESNG6J9//lG/fv3yzb++//771bFjR/Xv3189evRQ/fr1NWjQIJ+fNyIiQrNnz9aRI0d0yimnaOTIkXrkkUfy7XPeeefptttu00033aQTTjhBX375pR588MF8+1x44YU6++yz1bNnT9WpU6fQZctiY2M1f/587d+/X507d9bgwYPVq1cvvfjii6X7YRQiJSVFnTp1yvc1cOBAeTweffzxx6pRo4a6deum3r17q3nz5vrggw8kSZGRkdq3b5+uvPJKtWrVShdffLH69++vcePGSTJh/sYbb1Tbtm119tlnq1WrVnr55ZfL3d6ieCyrsBr54Ss5OVnVqlXTwYMHlZCQUOR+mZmZmjt3rgYMGFBg/gEQEMnJudUxk5OVWbmyvnn8cZ3x4INS69bSH3842z5UKLwHwmkcg3ASx1/oSEtL04YNG9SsWTNVrly57E/k9UorV5qOjgYNpK5dA9bD7Yvs7GwlJyfnVC+Hc4o7xnzNliwZBriF3ctdo4ZUtaqUmUlPNwAAQDBERkr/Lt0F+BunTQC3OGpouZRneHlysll3EgAAAEBIIXQDbnFU5XJJyoqNlVWlirlCbzcAAAAQcgjdgFscVblckqmc2aCB2SZ0AwAAACGH0A24RSHDyyXJql/fbBC6AQAAgJBD6AbcopDh5ZIkO3Tv3Bnc9gAAAISICrYgE4IoOzu73M9B9XLALQobXi7JYng5AABAoaKjo+XxeLRnzx7VqVNHHo/H6Sb5RXZ2tjIyMpSWlsaSYQ6xLEsZGRnas2ePIiIiVKlSpTI/F6EbcIOsLGnbNrNdVE83oRsAACCfyMhIJSYmauvWrdq4caPTzfEby7J05MgRValSJWxOJISq2NhYHXPMMeU6+UHoBtxg+3YpO1uKjs4N2f+ipxsAAKBo8fHxatmypTIzM51uit9kZmZqxYoV6tatm6Kjo51uToUVGRmpqKiocp/4IHQDbmDP527cWDr6LBo93QAAAMWKjIxUZGSk083wm8jISGVlZaly5cqE7jDABAHADYqYzy1RvRwAAAAIZYRuwA2KWC5MUm5P9549UhgNmwIAAAAqAkI34AZFLRcmSbVrS1H/zgTZtSt4bQIAAABQboRuwA2KGV6uiAipXj2zzRBzAAAAIKQQugE3KG54uSRRwRwAAAAISYRuwGmWVXxPt0ToBgAAAEIUoRtw2oEDUkqK2W7cuPB9CN0AAABASCJ0A06zh5bXqSPFxha+D6EbAAAACEmEbsBpxVUutxG6AQAAgJBE6AacVprQvXNn4NsDAAAAwG8I3YDTSiqiJtHTDQAAAIQoQjfgtJKWC5Py93RnZwe+TQAAAAD8gtANOM2X4eX16kkej5SVJe3bF5x2AQAAACg3QjfgNF+Gl0dHS7Vrm22GmAMAAAAhg9ANOCkjIzdEFze8XGJeNwAAABCCCN2Ak7ZtkyxLiokx63QXh9ANAAAAhBxCN+CkvEPLPZ7i961f31wSugEAAICQQegGnORL5XIbPd0AAABAyCF0A07ypXK5jdANAAAAhBxCN+AkXyqX2wjdAAAAQMghdANOYng5AAAAENYI3YCTyjq83LIC1yYAAAAAfkPoBpxiWWUL3UeOSMnJgWsXAAAAAL8hdANO2bdPOnzYbCcmlrx/bKyUkGC2d+4MXLsAAAAA+A2hG3CK3ctdv75UubJvj2FeNwAAABBSCN2AU0oztNxG6AYAAABCCqEbcEpplguzEboBAACAkELoBpxSmuXCbIRuAAAAIKQQugGnMLwcAAAACHuEbsApDC8HAAAAwh6hG3BKWYaX169vLgndAAAAQEggdANOSEuTdu0y2/R0AwAAAGGL0A04YcsWcxkbK9Ws6fvj7NB94IB05IjfmwUAAADAvwjdgBPyFlHzeHx/XPXqUkyM2d650+/NAgAAAOBfhG7ACWWZzy2ZgM4QcwAAACBkELoBJ5RluTAboRsAAAAIGYRuwAllWS7MRugGAAAAQgahG3BCWYeXS4RuAAAAIIQQugEn+GN4OYXUAAAAANcjdAPBlp3NnG4AAACggiB0A8G2Z4+Unm4qkScmlv7xhG4AAAAgZBC6gWCze7kbNpSio0v/eEI3AAAAEDII3UCwladyuZQbunfvlrKy/NMmAAAAAAFB6AaCrTyVyyWpdm0pMlKyLBO8AQAAALgWoRsItvIUUZNM4K5b12wzxBwAAABwNUI3EGzlDd0S87oBAACAEEHoBoKtvHO6JUI3AAAAECII3UCwlXdOt0ToBgAAAEIEoRsIpsOHpb17zTY93QAAAEDYI3QDwWT3cletKlWrVvbnIXQDAAAAIYHQDQRT3qHlHk/Zn4fQDQAAAIQEQjcQTP6oXC4RugEAAIAQQegGgskflcul3NC9c6dkWeV7LgAAAAABQ+gGgskflcslqX59c5mZKe3fX77nAgAAABAwhG4gmPw1vLxSJalWLbPNEHMAAADAtQjdQDD5a3i5xLxuAAAAIAQQuoFg8XqlrVvNdnmHl0uEbgAAACAEELqBYNm1y8zBjozMDczlQegGAAAAXI/QDQSLPZ+7USMpKqr8z2cXUyN0AwAAAK5F6AaCxZ/zuSV6ugEAAIAQ4GjoXrFihQYOHKiGDRvK4/Fozpw5JT4mPT1d999/v5o0aaKYmBg1bdpUb731VuAbC5SXv5YLsxG6AQAAANfzwxjXsktNTVXHjh119dVXKykpyafHXHzxxdq1a5cmTZqkY489Vjt27FB2dnaAWwr4gb+WC7MRugEAAADXczR09+/fX/379/d5/3nz5mn58uVav369atasKUlq2rRpgFoH+BnDywEAAIAKx9HQXVqffPKJTj75ZD3xxBN65513FBcXp/POO08PP/ywqlSpUuhj0tPTlZ6ennM9OTlZkpSZmanMzMwiX8u+r7h9gNKI2rRJHklZjRrJ8uG4KvEYrF1b0ZKUmqrM/fulqlX91laA90A4jWMQTuL4g9M4BkODr7+fkArd69ev16pVq1S5cmXNnj1be/fu1Q033KB9+/Zp8uTJhT7mscce07hx4wrcvmDBAsXGxpb4mgsXLix3uwFJ6r9+vSpJWrFxow7Nnevz44o7Bs+pXFlRaWla/v77Sm3UyA+tBPLjPRBO4xiEkzj+4DSOQXc7fPiwT/t5LMuyAtwWn3g8Hs2ePVuDBg0qcp++fftq5cqV2rlzp6pVqyZJmjVrlgYPHqzU1NRCe7sL6+lu3Lix9u7dq4SEhCJfKzMzUwsXLlSfPn0UHR1d9m8MkKRDhxRdq5YkKXPfPp96pX05BqPatZNn3TplLVokq1s3vzYZFRvvgXAaxyCcxPEHp3EMhobk5GTVrl1bBw8eLDZbhlRPd4MGDdSoUaOcwC1Jbdu2lWVZ2rp1q1q2bFngMTExMYqJiSlwe3R0tE8HsK/7AcWy513XqKHof+sR+KrYY7BhQ2ndOkXt3StxnCIAeA+E0zgG4SSOPziNY9DdfP3dhNQ63WeccYa2b9+ulJSUnNvWrl2riIgIJSYmOtgyoAT+rlxuo5gaAAAA4GqOhu6UlBStWbNGa9askSRt2LBBa9as0eZ/A8q9996rK6+8Mmf/yy67TLVq1dJVV12l3377TStWrNCdd96pq6++ushCaoAr+LtyuY3QDQAAALiao6F79erV6tSpkzp16iRJGj16tDp16qQxY8ZIknbs2JETwCUpPj5eCxcu1IEDB3TyySdr6NChGjhwoJ5//nlH2g/4zD6OmzTx7/MSugEAAABXc3ROd48ePVRcHbcpU6YUuK1NmzZU8UPoYXg5AAAAUCGF1JxuIGQRugEAAIAKidANBEOg5nTXr28uCd0AAACAKxG6gUDLypK2bTPbgZrTvX+/lGc9egAAAADuQOgGAm3HDsnrNeto2z3T/lKzplSpktneudO/zw0AAACg3AjdQKDZQ8sTE6UIP//JeTwMMQcAAABcjNANBFqglguzUUwNAAAAcC1CNxBogapcbiN0AwAAAK5F6AYCLVCVy22EbgAAAMC1CN1AoDG8HAAAAKiwCN1AoAVreDnVywEAAADXIXQDgcacbgAAAKDCInQDgXTggJScbLYJ3QAAAECFQ+gGAsnu5a5dW4qNDcxr2KF71y7J6w3MawAAAAAoE0I3EEiBHlouSXXrShERUna2tGdP4F4HAAAAQKkRuoFACvRyYZIUGWmCt8QQcwAAAMBlCN1AIAV6uTAb87oBAAAAVyJ0A4EUjOHlklS/vrkkdAMAAACuQugGAikYw8sleroBAAAAlyJ0A4HE8HIAAACgQiN0A4GSmSlt32626ekGAAAAKiRCNxAoW7dKliXFxEh16gT2tQjdAAAAgCsRuoFAyVtELSLAf2qEbgAAAMCVCN1AoASrcrmUP3RbVuBfDwAAAIBPCN1AoAQzdNtLhmVkSP/8E/jXAwAAAOATQjcQKMFaLkySKleWatQw2zt3Bv71AAAAAPiE0A0ESrCWC7MxrxsAAABwHUI3ECjBHF4uEboBAAAAFyJ0A4FgWcEdXi4RugEAAAAXInQDgbB/v3T4sNlu3Dg4r0noBgAAAFyH0A0Egj20vF49U+QsGAjdAAAAgOsQuoFACPbQcil32TBCNwAAAOAahG4gEIJduVyipxsAAABwIUI3EAjBrlwuEboBAAAAFyJ0A4HgxPByO3QfOiSlpgbvdQEAAAAUidANBIITw8urVpViY802vd0AAACAKxC6gUBwYni5x8MQcwAAAMBlCN2Av6WnSzt3mu1ghm6J0A0AAAC4DKEb8LctW8xllSpSrVrBfW1CNwAAAOAqhG7A3/LO5/Z4gvvadui2e9oBAAAAOIrQDfibE/O5bfR0AwAAAK5C6Ab8zYnlwmyEbgAAAMBVCN2AvzmxXJiN0A0AAAC4CqEb8DeGlwMAAAD4F6Eb8Dc3DC/fu1fKyAj+6wMAAADIh9AN+JNlOTu8vGZNKSrKbO/aFfzXBwAAAJAPoRvwpz17pPR0s1RYo0bBf/2ICKl+fbPNEHMAAADAcYRuwJ/sXu4GDaRKlZxpA/O6AQAAANcgdAP+ZM/ndmJouY3QDQAAALgGoRvwJycrl9sI3QAAAIBrELoBfyJ0AwAAAMiD0A34k5PLhdkI3QAAAIBrELoBf3JyuTAboRsAAABwDUI34E9uGl6+c6dzbQAAAAAgidAN+M/hw2adbskdoXvXLik727l2AAAAACB0A36zZYu5rFpVql7duXbUqyd5PFJWlrR3r3PtAAAAAEDoBvwm79Byj8e5dkRFSXXqmG3mdQMAAACOInQD/uKGyuU2iqkBAAAArkDoBvzFDZXLbYRuAAAAwBUI3YC/uKFyua1+fXNJ6AYAAAAcRegG/MVNoZuebgAAAMAVCN2Av9hzuhleDgAAAOBfhG7AH7Kzc5cMo6cbAAAAwL8I3YA/7NolZWZKERFSw4ZOt4bQDQAAALgEoRvwB3toeaNGZp1sp+UN3ZblbFsAAACACozQDfiDm5YLk3JDd1qadPCgs20BAAAAKjBCN+APbqpcLklVqkjVqplthpgDAAAAjiF0A/5gDy93S+iWcnu7d+50th0AAABABUboBvzBbcPLJYqpAQAAAC5A6Ab8wW3DyyVCNwAAAOAChG7AH9w8vJzQDQAAADjGBWsbASHu0CHpn3/MNqEb/uD1SitXmt9dgwZS165SZKTTrQIAAEAZELqB8tqyxVxWry4lJDjalHwI3aFp1izp1lulrVtzb0tMlJ57TkpKcq5dAAAAKBOGlwPl5cb53JJUv765JHSHjlmzpMGD8wduSdq2zdw+a5Yz7QIAAECZEbqB8rLnc7upcrlET3eo8XpND7dlFbzPvm3UKLMfAAAAQgahGygvt/Z026H74EHpyBFn24KSrVxZsIc7L8syUxlWrgxemwAAAFBuhG6gvNwauqtVkypXNtv0drufr78jfpcAAAAhhdANlJcblwuTJI+HIeahxP5d+Ws/AAAAuAKhGygvu6fbbXO6JUJ3KOna1VQp93gKv9/jkRo3NvsBAAAgZBC6gfLwenPn4bqtp1sidIeSyEizLFhxJk5kvW4AAIAQQ+gGymP7dhO8o6Jyl+hyE0J3aElKkmbOlCpVyn97tWrmdtbpBgAACDmEbqA87KHljRu7swfSDt07dzrbDvjuggukmBiz3aePuTzhBAI3AABAiCJ0A+Xh1srlNnq6Q8/WrdKhQ2b0xLPPmtu++kpKTXW2XQAAACgTQjdQHoRu+Nuvv5rLli2ldu3MsZWRIa1a5Wy7AAAAUCaEbqA87OXC3Fi5XCJ0hyI7dB93nKlY3ru3ub54sXNtAgAAQJkRuoHyCJWe7j17pKwsZ9sC3/z2m7k87jhz2auXuVy0yJn2AAAAoFwI3UB5uD10165tCrxZlrRrl9OtgS/y9nRLuaH7hx+kvXudaRMAAADKjNANlIc9vNytoTsiQqpXz2wzxNz9LKtgT3e9elL79mZ76VJn2gUAAIAyI3QDZXXwoJScbLbdGrol5nWHki1bTOXy6GhTSM3GEHMAAICQRegGysoeWl6rlhQX52xbikPoDh320PJWrUzwttnF1AjdAAAAIcfR0L1ixQoNHDhQDRs2lMfj0Zw5c3x+7BdffKGoqCidcMIJAWsfUCy3Dy23EbpDx9HzuW3dupl1u9evlzZsCH67AAAAUGaOhu7U1FR17NhRL730Uqked+DAAV155ZXqZQ+5BJxg93S7dbkwG6E7dBQVuqtWlU47zWyzdBgAAEBIcTR09+/fXxMmTNAFF1xQqsddf/31uuyyy9SlS5cAtQzwgdsrl9sI3aGjqNAtMa8bAAAgREU53YDSmjx5stavX693331XEyZMKHH/9PR0paen51xP/rfwVWZmpjIzM4t8nH1fcfugYovcsEERkryNGik7AMeJv45BT506ipKUvX27vBzP7pWdrajffpNHUmarVtJRvytPjx6KGjdO1uLFykpPN5XpA4j3QDiNYxBO4viD0zgGQ4Ovv5+QCt1//fWX7rnnHq1cuVJRUb41/bHHHtO4ceMK3L5gwQLFxsaW+PiFCxeWup2oGM786SfVkvTdvn3aMXduwF6nvMdg9fXr1V1S+oYNWhDAdqJ8quzapb6pqfJGRenzv/6StX59vvs9WVkaULmyovbu1apXXlFys2ZBaRfvgXAaxyCcxPEHp3EMutvhw4d92i9kQrfX69Vll12mcePGqVWrVj4/7t5779Xo0aNzricnJ6tx48bq27evEhISinxcZmamFi5cqD59+ig6bxVh4F9RN90kSTrx/PNlde7s9+f32zHYvr10112qfPCgBpx9dsB7SFE2nn9PiES0aaP+AwcWuk9Ez57S55+rW0aGsgcMCGh7eA+E0zgG4SSOPziNYzA02KOoSxIyofvQoUNavXq1fvjhB930b9jJzs6WZVmKiorSggULdNZZZxV4XExMjGJiYgrcHh0d7dMB7Ot+qGAyM6Xt2yVJUc2b51/eyc/KfQwmJkoyPaXRhw5JtWv7qWXwqz//lCR5jj++6N93nz7S558rculSRd51V1CaxXsgnMYxCCdx/MFpHIPu5uvvJmRCd0JCgn7++ed8t7388stasmSJZs6cqWZBGmoJSJK2bZOys6WYGKluXadbU7xKlUzQ3rvXFFMjdLtTcUXUbPZ63StWSBkZ5ncLAAAAV3M0dKekpGjdunU51zds2KA1a9aoZs2aOuaYY3Tvvfdq27Ztmjp1qiIiInT88cfne3zdunVVuXLlArcDAWdXLm/cODSGazdokBu627d3ujUojC+h+/jjzUme3bul//3PrN8NAAAAV3M0LaxevVqdOnVSp06dJEmjR49Wp06dNGbMGEnSjh07tNkON4CbhMpyYTaWDXO37Gzp99/NdnGh2+Nh6TAAAIAQ42jo7tGjhyzLKvA1ZcoUSdKUKVO0bNmyIh8/duxYrVmzJihtBfLZtMlcErrhDxs3SocPm+kKLVoUv689xJzQDQAAEBJCYFws4EJ2T3eTJs62w1eEbnezh5a3aSNFRha/rx26v/lG8rFiJgAAAJxD6AbKItSGl9evby4J3e7ky3xu2zHHSC1bSl6vtHx5YNsFAACAciN0A2XB8HL4kx2627XzbX/mdQMAAIQMQjdQWpbF8HL4V2l6uiXmdQMAAIQQQjdQWv/8I6Wmmu3ERGfb4qu8oduynG0L8vN6fatcnlfPnqaS+W+/cSIFAADA5QjdQGnZQ8vr1pWqVHG2Lb6yQ/fhw9KhQ862Bflt2CClpUmVK0vNm/v2mJo1pRNPNNuLFweubQAAACg3QjdQWqE2tFyS4uKkqlXNNj2j7lKayuV5McQcAAAgJBC6gdIKtcrlNuZ1u1Np53Pb8oZupgwAAAC4FqEbKC1CN/yprKH7jDOkmBhp2zZp7Vr/twsAAAB+QegGSsue0x1Kw8ul3NC9c6ez7UB+ZQ3dVaqY4C0xxBwAAMDFCN1AadHTDX/xeqU//jDbpQ3dEvO6AQAAQgChGygtQjf85e+/pfR002vdrFnpH2+H7qVLTYAHAACA6xC6gdJIT88NrYRulJc9tLxtWymiDG/HJ54oVa8uHTwoffedX5sGAAAA/yB0A6Wxdau5rFJFql3b2baUFqHbfco6n9sWGSn17Gm2GWIOAADgSmUK3Vu2bNFWO3xI+uabbzRq1Ci9/vrrfmsY4Ep5h5Z7PM62pbQI3e5T3tAtMa8bAADA5coUui+77DItXbpUkrRz50716dNH33zzje6//36NHz/erw0EXMWuXB5qQ8slqX59c/nPP1JamrNtgeHP0P3FF9Lhw+VvEwAAAPyqTKH7l19+0SmnnCJJ+vDDD3X88cfryy+/1HvvvacpU6b4s32Au9g93aG2XJgk1ahh1nWWWDbMDbKypD//NNvlCd0tW0qJiVJGhgneAAAAcJUyhe7MzEzF/PvhfdGiRTrvvPMkSW3atNEOhq4inIVq5XLJDIe3e7v5O3XeunUmKMfGlu8kjsfDEHMAAAAXK1PoPu644/Tqq69q5cqVWrhwoc4++2xJ0vbt21WrVi2/NhBwlVAO3RLzut3EHlrerl3ZKpfnZYfuxYvL9zwAAADwuzJ90nv88cf12muvqUePHhoyZIg6duwoSfrkk09yhp0DYcme0x2Kw8slQreb+GM+t+2ss8zl999L+/aV//kAAADgN1FleVCPHj20d+9eJScnq0aNGjm3X3vttYqNjfVb4wBXsSx6uuE//gzdDRqY5/n1V2npUmnw4PI/JwAAAPyiTD3dR44cUXp6ek7g3rRpkyZOnKg///xTdevW9WsDAdfYu9dU/fZ4TOGqUETodg9/hm6Jed0AAAAuVabQff7552vq1KmSpAMHDujUU0/V008/rUGDBumVV17xawMB17CHljdoIFWq5GxbyorQ7Q6ZmdLatWbb36Gbed0AAACuUqbQ/f3336tr166SpJkzZ6pevXratGmTpk6dqueff96vDQRcI9SHlku5oZslw5z1118meMfH++946tZNiow0VdE3bvTPcwIAAKDcyhS6Dx8+rKpVq0qSFixYoKSkJEVEROi0007TJrs3EAg34RS66el2Vt7K5R6Pf54zIUE69VSzTW83AACAa5QpdB977LGaM2eOtmzZovnz56tv376SpN27dyshIcGvDQRcwz6hFA6he/duyet1ti0Vmb/nc9uY1w0AAOA6ZQrdY8aM0R133KGmTZvqlFNOUZcuXSSZXu9OnTr5tYGAa9g93aG6XJgk1alj1oTOzjbBG84IVOju1ctcLl5sfscAAABwXJlC9+DBg7V582atXr1a8+fPz7m9V69eevbZZ/3WOMBVwmF4eWSkVK+e2WaIuXMCFbpPO02KjZX27JF++cW/zw0AAIAyKVPolqT69eurU6dO2r59u7Zu3SpJOuWUU9SmTRu/NQ5wlXAYXi4xr9tpGRmmkJrk/9BdqZLUvbvZZog5AACAK5QpdGdnZ2v8+PGqVq2amjRpoiZNmqh69ep6+OGHlc2QRoSjI0dM76EU2sPLJal+fXNJ6HbG2rVSVpYpfBaI9d5ZOgwAAMBVosryoPvvv1+TJk3S//3f/+mMM86QJK1atUpjx45VWlqaHnnkEb82EnDcli3mMj5eql7d0aaUGz3dzgpE5fK87Hndy5ebXvVQXVMeAAAgTJQpdL/99tt68803dd555+Xc1qFDBzVq1Eg33HADoRvhJ+987kAEpWAidDsrUPO5be3bm4J5e/ZIX38tde0amNcBAACAT8o0vHz//v2Fzt1u06aN9u/fX+5GAa5jz+cO9aHlEqHbaYEO3RERub3dzOsGAABwXJlCd8eOHfXiiy8WuP3FF19Uhw4dyt0owHXCoXK5jdDtrN9+M5eBCt1S/qXDAAAA4KgyDS9/4okndM4552jRokU5a3R/9dVX2rJli+bOnevXBgKuQOiGP6SnB65yeV52MbX//U9KTjZF2wAAAOCIMvV0d+/eXWvXrtUFF1ygAwcO6MCBA0pKStKvv/6qd955x99tBJwXjsPLd+6ULMvZtlQ0a9dKXq9UrZrUsGHgXqdpU6lFC/NaK1YE7nUAAABQojL1dEtSw4YNCxRM+/HHHzVp0iS9/vrr5W4Y4Crh1NNtLxmWkSHt3y/VquVseyqSvPO5A12Qr3dv6e+/zRDzc88N7GsBAACgSGXq6QYqlOzs3CXDwiF0x8RINWua7Z07nW1LRRPoImp5UUwNAADAFQjdQEl27TK9whERgR0SHEzM63ZGMEN3z56mN/2XXzi5AgAA4CBCN1ASe2h5o0ZSdLSzbfEXQrczghm6a9eWOnUy21QxBwAAcEyp5nQnJSUVe/+BAwfK0xbAncJpPreN0B18aWnSunVmOxihWzJDzL//3oTuoUOD85oAAADIp1Shu1q1aiXef+WVV5arQYDrELrhD3/+aeoD1KiRW8wu0Hr3lp580szrtqzAF28DAABAAaUK3ZMnTw5UOwD3CqflwmyE7uALZuVy25lnSpUqmUKAf/0ltWoVnNcFAABADuZ0AyUJx55uu6eV0B08wZzPbYuNlc44w2wzrxsAAMARhG6gJOEYuunpDj4nQrfE0mEAAAAOI3QDJWF4OfzBqdDdu7e5XLJE8nqD+9oAAAAgdAPFSkmR9u832+HY052SYr4QWEeOSH//bbaDHbpPOkmqVk06cMBUMgcAAEBQEbqB4mzZYi6rVZMSEpxtiz9VrSrFxZltersD748/TPXwmjWlunWD+9pRUVKPHmabed0AAABBR+gGimMPLQ+nXm4bQ8yDx4nK5XnZQ8yZ1w0AABB0hG6gOHYRtXCaz20jdAePU/O5bXboXrXKDHUHAABA0BC6geKEY+VyG6E7eJwO3a1bS40aSenp0hdfONMGAACACorQDRSH4eXwB6dDt8eTu3QY87oBAACCitANFKciDC/fudPZdoS7w4elDRvMtlOhW2JeNwAAgEMI3UBxGF6O8vr9d1O5vHbt4Fcuz8vu6f7uu9xl8AAAABBwhG6gKF6vtHWr2SZ0o6ycHlpua9hQatvWnABYtszZtgAAAFQghG6gKDt2SFlZZp1jO6CGE0J3cLgldEsMMQcAAHAAoRsoij20PDFRiox0ti2BYIfuffukjAxn2xLOCN0AAAAVGqEbKEo4z+eWpJo1pehos00xtcBxU+ju3t2cQPrrr9zjGwAAAAFF6AaKYi8XFo6VyyWzjFT9+mabIeaBkZIibdxott0QuqtVkzp3NtssHQYAABAUhG6gKOHe0y0xrzvQfv/dXNata6qXuwFDzAEAAIKK0A0UhdCN8nLT0HKbHboXLzaVzAEAABBQhG6gKPbwckI3ysqNofu006QqVaRdu3LbBwAAgIAhdANFsXu6w3VOt0ToDjQ3hu6YGKlbN7PNEHMAAICAI3QDhdm/Xzp40Gxv3Ch5vY42J2AI3YHlxtAtMa8bAAAgiAjdwNFmzZLatcu9PmCA1LSpuT3cELoD59Ch3NESbg3dy5dLmZnOtgUAACDMEbqBvGbNkgYPNvNd89q2zdwebsGb0B04v/1mLuvXN2uiu0mHDqaaekqK9M03TrcGAAAgrBG6AZvXK916a+EVne3bRo0Kr6HmdujevTu8vi83cOvQckmKiJDOOstsM8QcAAAgoAjdgG3lSmnr1qLvtyxpyxazX7ioW1fyeEzg3rvX6daEFzeHbol53QAAAEFC6AZsvg6xDqeh2FFRJnhL4fV9uYHbQ3evXubyf/8zw8wBAAAQEIRuwGYPtfbXfqGCed2B4fbQ3by51KyZlJUlrVjhdGsAAADCFqEbsHXtKiUmFn2/xyM1bmz2CyeEbv87eDB3qoJbQ7fEEHMAAIAgIHQDtshI6bnnCr/P4zGXEyea/cIJodv/7MrlDRtK1as72pRiEboBAAACjtAN5DVokBQfX/D2xERp5kwpKSnoTQq4+vXNJaHbf9w+tNzWs6e5/PnngsvkAQAAwC+inG4A4Crff2+KSsXHS7NnS3v2mJ7grl3Dr4fbRk+3/4VK6K5TRzrhBGnNGmnJEmnIEKdbBAAAEHYI3UBeCxaYy169cofehjtCt/+FSuiWzHG+Zo0ZYk7oBgAA8DuGlwN5zZ9vLvv1c7YdwUTo9r9QCt320mGLFpm16AEAAOBXhG7AduiQ9OWXZrtvX2fbEkx5Qzehq/wOHJC2bzfb7do52hSfdO0qRUdLmzdLf//tdGsAAADCDqEbsC1datYsbtHCfFUUduhOTzeBEeVj93InJkrVqjnbFl/ExUmnn262qWIOAADgd4RuwGbP565IvdySVLly7rJWDDEvv1AaWm5j6TAAAICAIXQDtoo4n9vGvG7/CcXQbc/rXrpU8nqdbQsAAECYIXQDkrR+vbRunRQVlbt2cUVih+6dO51tRzgIxdDdubNUtaq0f7+pZA4AAAC/IXQDUu7Q8i5dpIQEZ9viBHq6/ScUQ3fek00MMQcAAPArQjcgVdz53DZCt3/s3587WiAUKpfnlXfpMAAAAPgNoRvIzJQWLzbbFXE+t0To9pfffjOXxxxjhmuHEruY2qpVUlqas20BAAAII4Ru4JtvpORkqWZN6cQTnW6NMwjd/hGKQ8ttbdua4yAtLXe9egAAAJQboRuwq5b36SNFRjrbFqcQuv0jlEO3x8PSYQAAAAFA6AYq+nxuSapf31wSussnlEO3lDuv255uAQAAgHIjdKNi279f+vZbs12RQ7fd052cLB0+7GxbQlm4hO7Vq6V//nG2LQAAAGGC0I2KbfFiKTvbVJpOTHS6Nc5JSJCqVDHb9HaXzb590q5dZrttW2fbUlaJiVKbNlJ2tjzLlzvdGgAAgLDgaOhesWKFBg4cqIYNG8rj8WjOnDnF7j9r1iz16dNHderUUUJCgrp06aL59nxcoCzs46eiVi23eTzM6y4vu5e7aVMpPt7RppTLv73dniVLHG4IAABAeHA0dKempqpjx4566aWXfNp/xYoV6tOnj+bOnavvvvtOPXv21MCBA/XDDz8EuKUIS5bFfO68CN3lE+pDy23/FlOLIHQDAAD4RZSTL96/f3/179/f5/0nTpyY7/qjjz6qjz/+WJ9++qk6derk59Yh7P3xh7RlixQTI3Xr5nRrnEfoLh87dLdr52w7yqtHD8njkWftWjX973/liYuTevasuJX9AQAAysnR0F1e2dnZOnTokGrWrFnkPunp6UpPT8+5npycLEnKzMxUZmZmkY+z7ytuH4S2iM8/V6Sk7DPPlDc6WnLZ7zrYx2BEvXqKlOTdulXZLvtZhILIX35RhKSs1q1lhfDPz7NggSKjo+XJyFDHN9+U3nxTVqNG8j7zjKwLLnC6eahA+D8MJ3H8wWkcg6HB199PSIfup556SikpKbr44ouL3Oexxx7TuHHjCty+YMECxcbGlvgaCxcuLFcb4V6nvfee6kn6rXFj/T13rtPNKVKwjsGWBw+qnaRtq1frBxf/PNzq7DVrFCPpiwMHdCBEf34NvvpKnR9/vOAd27Yp8pJL9O3dd2tHly7BbxgqNP4Pw0kcf3Aax6C7HfZx1R+PZVlWgNviE4/Ho9mzZ2vQoEE+7T9t2jRdc801+vjjj9X73zmIhSmsp7tx48bau3evEhISinxcZmamFi5cqD59+ig6Otrn7wMhIj1dUXXrynPkiDJXr5Y6dHC6RQUE+xj0TJ2qqJEjld23r7z//W/AXy+s7Nmj6EaNJEmZ//wjxcU53KAy8HoVdeyx0rZt8hRyt+XxSI0aKeuvvxhqjqDg/zCcxPEHp3EMhobk5GTVrl1bBw8eLDZbhmRP9/vvv6+RI0dqxowZxQZuSYqJiVFMTEyB26Ojo306gH3dDyFmxQrpyBGpfn1Fn3iiqd7tUkE7Bv9dMi1i505FcMyXztq15rJZM0VXr+5oU8rsiy+kbduKvNtjWdLWrYr+3//MvG8gSPg/DCdx/MFpHIPu5uvvJuTW6Z4+fbquuuoqTZ8+Xeecc47TzUGoylu13MWBO6gopFZ24VC53NffO8cHAABAqTja052SkqJ169blXN+wYYPWrFmjmjVr6phjjtG9996rbdu2aerUqZLMkPJhw4bpueee06mnnqqdO3dKkqpUqaJq1ao58j0gRLE+d0F26N6zxxSV46yq78IhdNu/f3/tBwAAAEkO93SvXr1anTp1ylnua/To0erUqZPGjBkjSdqxY4c2b96cs//rr7+urKws3XjjjWrQoEHO16233upI+xGidu6UfvzRbJcwPaFCqVVLivr3PNyuXc62JdSEQ+ju2tVMMShq5IfHIzVubPYDAACAzxzt6e7Ro4eKq+M2ZcqUfNeXLVsW2AahYrCrQJ54olS3rrNtcZOICKl+fWnrVjOE+N853iiBZYVH6I6MlJ57Tho82ATswt6bJ06kiBoAAEAphdycbqDc8s7nRn7165tL5u36bvduad8+E1TbtHG6NeWTlCTNnCn9W4k9nzfeMPcDAACgVAjdqFiys3NDN/O5C6KYWunZvdzNm0uxsc62xR+SkqSNG5W1cKFWjx6t7LZtze379jnbLgAAgBBF6EbF8tNPpmcyLk46/XSnW+M+hO7SC4eh5UeLjJTVvbu2deumbLtmxqRJhQ85BwAAQLEI3ahY7KrlPXtKlSo52xY3InSXXjiG7jysiy4yJ6nWrjVreQMAAKBUCN2oWJjPXTxCd+mFeehW1arSxReb7UmTnG0LgIrB65WWLZOmTzeXXq/TLQKAciF0o+JITZVWrTLbzOcuHKG7dMKlcnlJRowwlx9+KCUnO9sWAOFt1iypaVMzIu2yy8xl06bmdgAIUYRuVBzLl0sZGVKTJlLLlk63xp0I3aWzc6f0zz9mubVQr1xenNNPl1q3lg4flj74wOnWAAhXs2aZZQu3bs1/+7Zt5naCN4AQRehGxWHP5+7XzyzvhILs0L1rl6n0juLZvdwtWkiVKzvblkDyeHJ7uxliDiAQvF7p1lsLL9ho3zZqFEPNAYQkQjcqDuZzl6xePROwsrJYIsoXFWFoue3KK6XISOnrr3O/bwDwl5UrC/Zw52VZ0pYtZj8ACDGEblQMmzdLf/xhhgH36uV0a9wrOlqqXdtsM8S8ZBUpdNerJ517rtl+6y1n2wIg/Pj6P4f/TQBCEKEbFYPdy33qqVL16o42xfWY1+27ihS6pdwh5lOnmvoIAOAv9v8ef+0HAC5C6EbFkHc+N4pH6PZNRalcnlf//ub42LtX+vRTp1sDIJx07SolJhZfc6VxY7MfAIQYQjfCn9crLVpktpnPXTJCt2+2b5cOHjTznFu3dro1wREVJQ0bZrYpqAbAnyIjpeeeK7yQmu2GG8x+ABBiCN0If99+Kx04YIaVd+7sdGvcj9DtG7uX+9hjpZgYZ9sSTFdfbS7nzy++6BEAlFa7dqb2ytGqVDGXzz9vTngCQIghdCP82fO5e/UyPXUoXv365pLQXbyKNrTc1rKl1K2bWVJuyhSnWwOEJ69XWrZMmj7dXFaUZbLuvde8twwcKC1dKk2bZi63bzfvtTt2SElJUlqa0y0FgFIhdCP8MZ+7dOjp9k1FDd1SbkG1yZNZzx3wt1mzpKZNpZ49pcsuM5dNm5rbw9kXX0hz5pie7v/7P6lHD2nIEHNZvbq5r3p1s2zhDTcUPwwdAFyG0I3wduCA+QctMZ/bV4Ru31Tk0D14sFS1qrR+vbR8udOtAcLHrFnm7+voqRvbtpnbwzV4W5Z0111m+6qrzDDzox17rPTBByaUT54svfhicNsIAOVA6EZ4W7LEDMtr3Vpq0sTp1oSGvKGbnoTCWZb0229muyKG7thY0wMlUVAN8BevV7r11sLfd+3bRo0Kz6Hmn3wiffmlmbs9blzR+/XtKz3xhNm+7TbzPx4AQgChG+HNns9NL7fv7NB95IiUnOxsW9xq61bzs4mKklq1cro1zrCHmH/0kRlRAqB8Vq4svjihZUlbtpj9wklWlnTPPWZ71CipUaPi9x89Wrr8cnPy4eKLpQ0bAt5EACgvQjfCl2Uxn7ssYmOlhASzzRDzwtlDy1u2lCpVcrYtTuncWTr+eFPQaNo0p1sDhD5f32/D7X158mTpjz+kWrWku+8ueX+PR3r9denkk6V9+6RBg6TU1IA3EwDKg9CN8LVunbRxoxQdLXXv7nRrQgvzuotXkedz2zye3N5uhpgD5We/7/prv1CQmio99JDZfuABqVo13x5XpYo0e7ZUr57000/S8OFMhwLgaoRuhC+7l/vMM6X4eGfbEmrsZcM++qhiLVfjK0K3cfnl5qTW999La9Y43RogtHXtKiUmFn2/xyM1bmz2CxcTJ5qTu02bSv/5T+kem5ho/kdFR0szZ0qPPhqIFgKAXxC6Eb6Yz102s2ZJ33xjtl96qeIsV1MahG6jdm0ztFOS3nrL0aYAIS8y0oTQoliW9OyzZr9wsGeP9PjjZvuRR6SYmNI/xxlnmP9TkvTgg9Knn/qvfQDgR4RuhKeMDGnpUrPNfG7f2cvVHDmS//ZwX66mNCp65fKj2UPM333XzO8GUHb2KKOiHP3eHMomTJAOHZJOPFG69NKyP8811+Su2z10qPT77/5rIwD4CaEb4emrr6SUFKlOHaljR6dbExoq8nI1pbF5szm2oqNNIbWKrndvM+T1n3+kOXOcbg0Q2p56ylyOGGFOHE+bZi7Hjze3jxpleohD3fr10iuvmO3HHzdrb5fHxIlSt24mxJ9/PisqAHAdQjfCkz2fu2/f8v8zrygq6nI1pWUPLW/VygTvii4y0hQxkiioBpTHX39JH39stu+4Q+rRQxoyxFzec4/UoYOp1n3rrU620j8eeEDKzJT69DEn7sorOlqaMUM65hjzcxwyhBPEAFyFNILwxHzu0quoy9WUFvO5C7rqKnO5aJFZMQBA6T37rDm5ee65Ups2+e+LjjYntSIipOnTpc8+c6aN/vDdd+Z7kHLndPtD3bpmtE2VKtK8edJ99/nvuQGgnAjdCD979phqypI5iw7fVMTlasqC0F1Qs2ZSr15me/JkZ9sChKK9e3P/du64o/B9Tj5Zuu02s3399VJycnDa5k+WlbsW99ChUqdO/n3+Tp1yf45PPJEb7gHAYYRuhJ9Fi8w/9g4dCIilYS9X4/EUfn84LldTFoTuwtkF1SZPZlgnUFqvvGIKEZ50kpmbXJTx46Xmzc1UoHvvDV77/GXBAmnxYqlSJVNILRAuucQMx5ekq6/OPQkPAA4idCP82PO5qVpeOpGR0nPPme2igvfEieGzXE1ZZGdTubwoF1wg1ahh5v0vXux0a4DQkZYmvfCC2b7jjqLffyUpNlZ6/XWz/fLL0qpVgW+fv2Rn5/Zy33ijWYoyUCZMkPr3Nz/bQYOk3bsD91oA4ANCN8KLZTGfuzySkqSZM6VGjfLfHhdnbk9KcqZdbrFpk3T4sOmlOfZYp1vjLpUrm+GiEgXVgNJ45x0zLeqYY8zSjCXp1cv04ErSyJGhs1Tfe+9JP/4oVasm3X9/YF8rMtJUfm/VypwIHDzYLCUKAA4hdCO8/PKLKfRVpYp05plOtyY0JSWZYlhLl0pjxpjbLIv58VLu0PLWraWoKGfb4kZ2EJgzx1RZBlC87GzpmWfM9qhRvr+vPPWUWdP7zz8DN0zbn9LSTMVyyQz9rlUr8K9ZvbqpBp+QYFbdCIeq7wBCFqEb4cXu5e7e3fS8oWwiI80yNWPHmiq6hw9TkEZiaHlJOnUyXxkZ0rvvOt0awP3mzpX++MP0/o4c6fvjatSQXnrJbD/+uPTTT4Fpn7+89JK0ebMZRRXM8Numjelh93ikV1+VXnsteK8NAHkQuhFemM/tXx5P7gfBN95wti1uQBG1ktkF1SZNMiMkABTtqafM5bXXSlWrlu6xSUnmKyvL/N1lZfm/ff7wzz/SI4+Y7fHjzUi0YDr33NzRADfdFFrz4AGEDUI3wseRI9KKFWab+dz+c+WVZo3Y1aulNWucbo2zCN0lu+wyKSZG+vlnc8wAKNzq1dLy5WZI+S23lO05XnzRDKNevTq3EKbbPP64Cd7t2pn/J064917poovMiYkLLzTzvAEgiAjdCB8rVkjp6WbZq7ZtnW5N+KhTx1Smlip2b3d2tvT772ab0F20GjXMh1pJeustZ9sCuNnTT5vLIUPM/62yaNAgt7f8wQelv//2T9v8ZcuW3JMB//d/ztXC8HjMcoYdO5pK5oMGmRP1ABAkhG6ED3s+d79+xS+5gtK75hpz+e67Umqqs21xysaNuZXLmzd3ujXuZg8xnzbN/MwA5LdpkzRjhtm+/fbyPdfVV0tnnWVC5LXXumtax0MPmSJqXbuaYd5OioszRR5r1zZrd19zjbt+VgDCGqEb4cOez83Qcv876yypWTMpOTn3g2JFYw8tb9OGyuUl6dEj93j56COnWwO4z3PPSV6v1Lu36X0tD4/HrN1dpYq0ZIl7Rpj88ov09ttm+4kn3HEyvGlT8z8sMtIUWLNHGwBAgBG6ER62bTOhyOMxH2LgXxERFFRjPrfvIiJylw9jzW4gvwMHct9H77jDP8/ZooUpUiaZnvMdO/zzvOVxzz1mWs6FF0qnneZ0a3L16CFNnGi2774794Q9AAQQoduNvF5p2TKzRNOyZeY6imcPLe/cWapZ09m2hKurrjK9A19+mRtAKxJCd+kMG2ZOgi1fLq1b53RrAPd44w0pJUU6/nj/jswaNUo66STp4EFTpdtJy5dLn31m/mc8+qizbSnMjTeaaTDZ2dKll0p//eV0iwCEOUK328yaZYY/9expqgD37Gmuz5rldMvcLe98bgRGgwa5c/LefNPZtjiB0F06jRvn/j26Zbgr4LSMjNzCYrff7t8h11FRZmRJVJT5zODU5wbLku66y2xfe63UqpUz7SiOx2PWDu/SxYw8OP98Mx0GAAKE0O0ms2ZJgwdLW7fmv33bNnM7wbtwXq+0cKHZZj53YNkF1aZONcVxKgqvl8rlZWEXVHv7bfeuIQwE0wcfmP/pDRqYquX+1rFjbuC98UazVFewzZwpffONKVw2ZkzwX99XMTGm5kTDhub9/YorTM83AAQAodstvF7p1lsLr6Rp3zZqFEPNC/PDD9K+fVJCgnTqqU63JrydfbZZ2mb/fmn2bKdbEzwbNpiTDJUrU7m8NM47z1QK3r6deZOAZeUW7rr5ZhP6AuHBB6XWraWdO6U77wzMaxQlM1O67z6zffvtUv36wX390mrQwPwvi4mRPvlEGjvW6RYBCFOEbrdYubJgD3delmXWu1y5MnhtChX2h/mzzpKio51tS7iLjMztvaxIBdXyVi6PjHS2LaGkUiXTeyRRUA1YvFj68UfTA3zddYF7ncqVc9+fJ00yFc2D5Y03TA2HunX9VyQu0E45xVR/l6SHH2bFBQABQeh2C18rjbqhIqnbMJ87uK6+2syHW7q04hTIYj532dknaT79VNq1y9m2+IpilgiEp54ylyNGBL7gZ9eu0n/+Y7avuUY6fDiwrydJhw5J48aZ7TFjpKpVA/+a/nLlldJtt5ntYcOkNWvkWb5cjVaskGf5ct4DAJQbodstGjTw734VRXKyqaYtMZ87WI45xgwzlypOQTVCd9kdd5yZ9pGVJb3zjtOtKRnFLBEIv/xiRmVFRJipYsHwf/9npgOtXy899FDgX+/pp6Xdu6VjjzUF1ELNE0+YJUdTU6WTT1ZUnz46+ZlnFNWnD+8BAMqN0O0WXbuaf47FVTKNiDBzl5Fr2TLzYf7YY5lrG0x2QbUpU8wcvnBH6C6fvGt2F1a3wi0oZolAsedyX3ih1KxZcF4zIUF69VWz/cwz0urVgXutnTtze/IffTQ0p3pFRUmXX262j+7Z5j0AQDkRut0iMjJ3GZGignd2tnnTHzJE2rMneG1zM3s+N73cwXXuuVK9ema48KefOt2awPJ6pT/+MNuE7rK59FIpNtb8HL/6yunWFI5ilgiUHTuk994z27ffHtzXPucc85khO9sMaw/USdLx400P8SmnmM8pocjrlR54oPD7eA8AUE6EbjdJSjJLbTRqlP/2xo3N3MJ77jHh/P33zYf/Dz90d69RMDCf2xnR0dJVV5ntcC+o9vffUnq6VKVK8Hqowk1CgnTRRWbbrWt2U8wSgfLCCybsnnmmMytsPPecVKuW9NNP0pNP+v/5167NLUT2xBP+XXs8mHgPABBAhG63SUqSNm40RaqmTTOXGzaYnqLHHpP+9z/p+ONNT/cll5gzyjt3Ot1qZ6xfbwp5RUVJPXo43ZqKZ+RIczl/vrRpk7NtCSR7aHnbtmaKB8rGLqj2wQdSSoqzbSkMxSwRCCkp0iuvmG2nqnnXqSNNnGi2x4+X/vzTv89/332m9/ecc6Tu3f373MHk69/2DTdIjzwiff01vd4AfMYnSDeKjDQhcsgQc5l3iaKTT5a++85UBo2KMvOLjjtOevfditfrbfdyn3666UlDcLVoYZZpsyz39l76A/O5/ePMM6VWrUwI+fBDp1tTEMUsEQiTJ0sHDkgtW0oDBzrXjqFDpf79zaidkSPNcHN/+N//zBJbHo8p3BbKfP3b/v13Mwz9tNOk2rVNZ8krr0h//VXxPocB8BmhOxRVqmSW5fj2W6lTJ2n/frMW7nnnmWIfFQXzuZ1nF1SbNMkUtAtHhG7/8HjyF1Rzm5NOkmJiit+nXj1T9BLwRVaW9OyzZnv0aGdHyng8JhjGxUmrVkmvvVb+57Qs6a67zPawYWYUXigrqaCtx2OC+UsvmaBdvbo5oTJ7tun9btXKVDkfOdJMA6T2DoA8CN2h7IQTzPCmCRNMEP/vf00weOut8D/bmpkpLVlitpnP7ZwLLjBzBbdtk+bNc7o1gUHo9p8rrzQjd7780vQWucXBg9KAAaYXsDhpadLmzcFpE0Lf7Nlmeljt2ubYd1qTJmaamiTdfXfx85d98dlnZn5z5cpm2HqoK66grX39xRdNwP7oI2nvXvMZ7JFHzKjE6Gjz/jBpkhmpWLeu6Ri5804zMi8Ya6UDcC1Cd6iLjpbuv1/6/nupc2fz4XHECDOMLJw/HH79tVmju1Yt808NzoiJyf0wGY4F1bKycuc/ErrLr0EDE24l90xJ2LPHrMW9apVUrZr5AJ2YmH+fRo1MYDl40JzkowcLJbGs3CW0brjBVO93gxtukLp0kQ4dkv7zn7KfoPd6TXFXSbrlFlPwNRwUVdA2MdHcnpSUe1tkpKnWft99pv7OP/9In39uKtR37Gj2WbPGHAf9+kk1a0q9epkTH6tXlzwf3Os1y6JOn24umT8OhDRCd7g47jjTe/TEEyYIzZ9vhnq99pr/5m65iT2fu0+f/HPeEXz2EPPPPpO2b3e2Lf62bp2UkWE+MDdp4nRrwoNdUG3qVOfXeN+6VerWTfrhB1Nsatky8wH66GKWmzaZpc6aNDHzNs891yyPBBTliy+kb74x/49vvNHp1uSKjJTefDN3dNwHH5Tted5+24wCqlEjN3yHi38L2mYtXKjVo0cra+FCM2Ihb+AuTFycdPbZJmSvWWOK3E6bZlb6SEw0I2mWLDHvMZ07m57wiy4yn9P+/jv/c82aZYaq9+wpXXaZuWzalHXCgRBG6A4nUVFmGNOPP5riYocOSddfL/XubSp9hxPmc7tH27bSGWeYs/CTJzvdGv+yh5a3a0flcn8ZMMDMjd6925yoccq6daa42x9/mA/EK1eaKTtS4cUsGzQw7zu1apkwdfHFzp80gHs9/bS5HDbMhCs3adfOjJCTpJtvNsOkS+PwYVPMVTLPU6OGf9vnBpGRsrp317Zu3WR17162k/v16pn3kLfeMiMP//jDDE8//3xT/HX/ftN7fv310rHHSs2bS9dea6rcDx5ccPj/tm3mdoI3EJL4FBmOWreWVqwwS4RUqWJ6atq3N2uFhkOv9/79poicROh2i7wF1cLhGLMxn9v/oqNNEJGcK6j2yy+maNKmTebD7qpV5n2zJK1bm97BKlWkuXOl664L//oZKL21a6WPPzbbt93mbFuKcs89ZjTc3r2lb+Pzz5sAeMwx7urFdzOPx7x/3HijNGeOtG+fGT0zfrwZbRMVZXrT33jDnLAp7H3Fvm3UKIaaAyGI0B2uIiOlW2+Vfv7ZrJt5+LCZd9W9uxkeGcoWLTL/fI47ruC8KzjjoovMfNgNG6TFi51ujf8QugPDrmI+d27wpyR88415H9y505yMXLmydFMHTjvNLHkWGWlGdjz4YODaitD07LPmf9TAgVKbNk63pnCVKplh5h6PWXLU10KY+/blLg02YYIpoobSi4oy7yUPPigtX27mg3/2mXThhcU/zrKkLVvM+xaAkELoDnctWpg5RC+/LMXHmx6dDh3MmdRQPVNqz+emarl7xMaadWCl8CqoRugOjNatzdDu7GwzNzRYli0zhYz275dOPdVcr1+/9M9z7rm5Sy498oh5fwUkU2RvyhSzfccdjjalRKeeak7OS2bUxqFDJT/mkUdMQcGOHXPf81F+8fFm6k1Jodu2Y0dg2wPA7wjdFUFEhKlS+ssvpvBYWpr5MHDGGdJvvzndutKxLOZzu9W115rLOXPCo7pzZqYZJioRugPBLqgWrCUO//tfU+QoJUU66yxp4UJTTbisRozIXSbpppuYZwnjlVfM/9iTTw6NNd0nTDAFujZvzp3nXZSNG80a1ZL0+OPUuQiEBg38ux8A1+AdsyJp0sQE1jffNEU8vv7aLLf12GNmaaRQ8McfprhITIyZBwX36NjRVGTNzAxu72Wg/PWX+V7i483cRfjX4MHmZ7tunalBEUjvv2/WlE9PN0N+P/tMqlq1/M/7wAOmCJJlmQrDgf4+4G5HjphCWZI5sX30Ws9uFBeXOzrpxRfNPOOiPPigWc3hrLM46R0oXbuawo5FHTsej1meLRRO6ADIh9Bd0Xg8pofm11/NUKaMDLN8xamnSj/9ZPZx89qQdi93t26mmBHcxS6o9uaboV1gyus183Yl8wEonIrDuUV8vHTppWY7kGt2v/66CcRZWebyo4/8Nw/V4zFBZdAgE+jPO8+MKELF9O67ZpRPkya+DxN2g969peHDzXv2iBHmWD7amjXSe++Z7SeeCI0TCqEoMlJ67jmzXdjP2LJMkVyWSgVCDqG7okpMNMMtp041y318/70ZDnfJJeYDg1vXhmQ+t7tdeqnpOfnzz9At9GKvjzpunLn+xx/u+hsIJ/YQ8xkzzDxRf3vqqdwK49dfL73zjqme7k+RkWYt3jPOMN/D2WebQkeoWLKzc5cJGzXKFMoKJU8/bZa4+v136dFHC95/993m7+jSS6WTTgp++yqSpCSzlFhhhWIjI83SYgBCDqG7IvN4pCuuML3egwaZobQffmiWAsnLLWtDpqWZnneJoW1uVbWqWZdUCs2CarNmsT5qMJ16qlkz+MgRMwTcXyzLDIW9805z/a67TLGzQM1BrVJF+uQTs2b9tm0meO/fH5jXgjt99pk52VitWu7JpFBSs6ZZVlQyU87yjthYtMic8I6ONoXUEHhJSWYO/dKl5qTekiVmiozXaz63FTYaAYCrEbphCnLMmFF0USG3rA35xRfmw3mDBmZ9UbiTPcR85kyzDEqo8HpNJV/WRw0ee7qL5L81u7Ozze9xwgRz/dFHTdGnQA+HrVnTLLvUqJEpUHn++eb9ChWD3ct93XX+qRfghMGDzXGbmWn+LhcvNkPK//Mfc/9//kMvazBFRko9epgT2T17mhUT6tQxJ0TGjnW6dYHn5qmOQBkQumGsWlV8z4wb1obMW7Wc+WTu1bmzWZYuLc3McQwVK1cW7OHOyw1/A+HoiitMD9q330o//1y+58rKMmHB7rF78UXp3nvL30ZfHXOMCd7Vqpn31Msu44NiRbB6tVlrOSpKuuUWp1tTdh6PqU5epYpZz753b+nyy02xQ49HOuEEp1tYsdWpY2pUSGZe/ZdfOtueQLKnebl1qiNQBoRuGL6u+ejk2pDM5w4NHk9ub/cbb4ROQbVQ+BsIR3XqmAJkUvl6u9PTzXzTKVNMD9HUqdKNN/qliaVy/PFmqHlMjFk+76abQudvAGVj93Jfdlnh83BDyddfFz5Cwy6yRuhx1qBB0pVXmhE9w4ZJqalOt8j/mOaFMEXohuHrmo/TpzuzBvPOndKPP5pA17t38F8fpTN0qKkQ/fPP5kNcKGB9VOfYQ8zfeadscxVTU01w/+gjqVIlM7Xhiiv828bS6NbNDMv1eKRXX2UebDjbuNFMz5Kk0aMdbUq52VNsisMUG+c995wphrtunXTPPU63xr+Y5oUwRuiGUdLakLZPP5VatjRLVmRmBqVpkqSFC83liSeanjG4W40a0kUXme1QKajWtWvRdQ0k1kcNpL59TQ/h/v2ml7g0Dh40o18WLJBiY01Bq0GDAtLMUrnwwtxh7g8+6L8563CX554zAaBPH6ljR6dbUz5MsQkN1avnvp+8+KKZex8ufD0G7aK6geT0nHKvV57ly9VoxQp5li/nREMYIHTDKG5tSI/HfD38sNSpk/mQe9ttZt6uPc860PLO50ZosIeYv/++lJzsbFt8sXatdPhw4ffZfxOsjxoYkZFmnWCpdOF0zx4z1++LL8wH0UWL3DUS5sYbpfvuM9vXXWeWaUT4OHBAevNNs33HHY42xS+YYhM6+vbNLXB31VWBWXLRCb4eW337Ss2amVFFl19u3mdffVWaO9cUmivvz8PpOeX/vn5Unz46+ZlnFNWnD3PawwChG7mKWhsyMdHc/sADptjRG2+Y3uY//jBL45x3nvTXX4FrV3Z2bk8387lDx5lnSm3amCA7fbrTrSleSorpmUxLM3NyExPz32//DSQlOdO+iuDqq83lggXS5s0l7791q/nA9cMPUt26pieiS5eANrFMJkwwJxS8Xunii6X//c/pFsFfXn/dvHe0b296ukMdU2xCy5NPSi1amJ7fUaOcbo1/+HpsZWebqR0rV5qpPI89Zk5CnHOO+XusXt0UtGzf3tz2n/+Yfd59V1qxwjy2qNGaTs8pd/r1ETBRTjcALpOUZJYMWbnSnHFs0MAMp7V79yIjpZEjzR/+ww9Lzz9vhpzPm2fe9B94QEpI8G+bfvxR2r1bio9354dqFM7jMcfKHXeYEzXXXed0iwpnWaZX/vffpYYNTW9p7dpF/w0gMJo3N70JS5eaYmhjxhS977p1pkd70yYz5H/RIqlVq6A1tVQ8HhPOdu2SPv9cOvdc0zPfurXTLUN5ZGTkjg67/fbwWFHDnma2bVvhc2o9HnM/U2zcIS5Oevtt8/uYMsWs420XpQxVZ55pPuulpBR+v8djOoZWrTLH6ebNuV9btuRu799vRtj98kv+NeePfq6GDc2qE40bm8vERPPZtqg55R6P+axr/5yzsor+ysws/v7C9snIMCsglPT655/PZ5IQROhGQfbakMWpXt1UbL32WjPU/PPPzVnXqVPN2cRhw6QIPw2ksKuW9+xpiiQhdFx5pVmy6bvvTI9kp05Ot6igF180Q+CjoqQPP5Tq1TO3l/Q3AP8bMcKE7smTzQm8wt5DfvnF9Cru3GnqSyxaZD4suVl0tCm21bOnGS3Ur59Z7qdhQ6dbhrL64ANp+3bzOxwyxOnW+Ic9zWzwYPPhPu8Hf6bYuNMZZ5gT208+aU4en366OWkcqp56qvjALZljtEkT83X66YXvm5KSG8LzhvG8AT0jwwT3bdukr77yrX32nPLo6NJ/b/6Qt64Cn1FCDqEb5dO6tZlDM3euCd9r15phoi+/bHrB/dEzzXzu0FWnjjn7/uGHprf75ZedblF+X32VW3H4ySfNBxg4JynJDAncuFFasqTg/OxvvpH69ze9GB06mBNy9kkSt4uLM0XezjjDTMcZMMCs7VytmtMtQ2lZlgkHknTzzeF1MtieZnbrrfmHtyYmmsDNFBv3GT/efAb79VczjPrDD0Nz5MWkSeYkvWTmqS9cWPZjMD5eatvWfBUmO9vUBDk6jH/5pfk/Ux7R0eYkvv119PWivqKjpb17ze+xJK++akaHuf2EM/KzKpiDBw9akqyDBw8Wu19GRoY1Z84cKyMjI0gtCwPp6Zb11FOWVbWqZZmPJZY1dKhlbdlS9udMSbGs6GjzXGvX+q+tISBsjsFFi8zvLyHB/D7dYvduy2rUyLTtoossKzvb6Ra5imPH3w03mN9Jz56WNW2aZS1dallZWZa1ZIllxceb+047zbL27w9uu/zl778tq149832cdZZlpaU53SLXcu174IIF5vcXFxe6x2FJsrLM317ev8EKxrXHX2G++86yoqLMcfnee063pvTmzLGsiAjT/nvuMbc5cQwuXZr7+bW4r48+sqx9+yzr4EHLSk01n3+93uC9vmRZHo/5HzJlimUlJ5f/tVFmvmZLCqnBfypVMnPb/vrLDBP1eEyBi9atzTq1aWmlf85ly8ycl6ZNpWOP9XeLEQw9e5ozssnJuevZOs3rNUNCt20zx+ekSaHZMxCOmjY1l0uX5laNrVfPjHRJSZHOOsv0gNSo4Wgzy6x5czMdJz7e9OYPG2Z6XRA6nn7aXI4cGbrHYUnsaWZDhphLhpS724knmqUJJbNqwvbtzranNFaskC691LwPXn219Oij5nYnjsGSls+1lw49/3yzxGhCglmqslIl/0yp9OX1a9aUunc30XvJElOos3596YorzOgvlhZzLUI3/K9ePbOMyrffmvk2hw+b+Zlt25qqi4UViCiKPZ+7Xz9CUaiKiDAfTiX3rNn90ENmbdO4OHNMVq3qdIsgmd/F3XcXvH3fPlNk5uSTzRDt+Pjgt82fOnUy32t0tJkbfPvtpXtfhHN+/tlMeYqIMEOwAbe4917zHnnggOn4CIX3lJ9+MkXJ0tLM5WuvOftZr6Tlc6XA1jXw5fXfeMN0SG3YYFbHaNXKfM5+913zWfmYY6S77iq6gBwcQ+hG4Jx0kqkwOW2aqTa5caNZlqlXL/NG6wvmc4eH4cPNP5Mvv/RtvlIg/fe/ZuSFZP55tWvnbHtgeL0mxBT3QXHnTucK2Phbnz6m4rBkPsTZvadwt2eeMZeDB5t1ggG3iI42xWxjYsyKMm45yV2UDRtMSDx40PTw2gVNnVbS8rmBrmvg6+s3bSrdf79Zvvd//5NuuMH0gm/fbmrUtG9vRkBMnGhWz4DjCN0ILI/HDA36808z9Ckmxgwb7dTJDIHat6/ox27aZB4XGWmGlCJ0NWggDRxott9807l2rF9vhmBJpgBSuFQdDgcrVxZcl/RoW7ea/cLFZZflFuS6807TUwH32r7dTJmSzOgEwG3ats0dnj16tPmf50a7d5vOlJ07TTj85BOpShWnW5UrKcl0FC1dajqOli41JwmCVUjw39fPWrhQq0ePVtbChUW/vscjnXqq9NJL5j1q1ixp0CBzEuaHH0yR40aNzHKVH35Ytqme8AtCN4IjLs5U2PzjD9NDkJ1tKlm3bGmWbMrKyt3X6zVDZyZMMNdPOcUsUYbQds015nLqVGfe9I8cMcfegQPSaaflhh24w44d/t0vVNx+u/lQJJmKvfaUGrjPCy+YGiNdu5r/S4AbjRoldesmpaaa9xS31YxITjarUKxbZ3pr581z52c8p+saREbK6t5d27p1k9W9u2+vHxNjVoyZPdsE8BdfNO9VXq+ZmnXJJWb+97XXmpGoxY0ssz+LT59uLpkrXm6EbgRX06ammNaSJebs5j//mB7HE04w6+3OmmX26dkzt0f011/N7Qht/fqZAiT795t/CMF2883mrG/t2uYYDKdlfsJBgwb+3S+UPPWUKSSUlWWm4Hz3HR943CYlxSzTI5l1kQG3iogwU1fi4kyRsokTnW5RrvR0Ewq//94sKbpggVnrHv5Xu7YZUfr116bD6/77zXzvgwfN1IOuXaUWLaSxY6W//87/2Lyfxe2Cpk2b8lm8nAjdcEbPnuZN95VXpFq1TLDu08d84Dx6iGlysumh5I89tEVGmsqkUvDnmk2aZL4iIsy8scTE4L4+SuZr1diuXYPbrmCwPyT36pVboT0xkQ88bvLWW2aUTKtWZpgm4GbNmuXWH7jvPun3351tj2ROHF5xhel0iY83qzi0bOl0qyqG1q3N6NENG8xQ+auuMr+DDRukcePM6kBnnmkK2U2daj5zH/1ZfNs2PouXE6EbzomKkq6/Xlq7VrrpppL3HzWK3p5Qd/XVJjwtXWqWlguGH34wZ3sl6eGHTbCB+zhdNdZpMTG5vQvJyWauY1584HFOVpb07LNme/Ro/ywNBATaNdeYYdzp6dKVV5qpEU6xLOmWW3JHmc2ZY4rtIrgiIsxQ+bfeMsXV3nvPjEKMiJC++MJ8Jh82rPBh5/ZtfBYvM/5zwHk1a5oe7uJYlrRlS3gVUaqIjjlGOvtssx2Mgmr//GOCSnq66Z26557AvybKzumqsU6LizPHamH4wOOc2bNNUaXatU14AUKBx2P+z9aoIa1eLT32mHNtGT/e1PHxeEzBSE5+Oy821oykmjfPfL5+8klz0rc4fBYvF0I33KGiFlGqiOyCalOmSBkZgXud7Gxzxnb9evOPZOpUeqhCgdNVY520cmXx73F84Ak+yzIfRiUzYsZNFZaBkjRsaIppSWak1/ffB78Nr75q5g1Lpi0XXRT8NqB4DRuaWhX2cqol4bN4mbhgQTxAFbuIUkVz7rlSvXpmaNOnn5Y8yqGsHn/cPH9MjPTRR+ZsP0KDXTW2ouHko3t4vebkxtKl0rffmveRG25wulVA6Q0ZYkZrzJxpRmqsXi1Vrhyc1545M/fvZswY/obczteidnwWLxO6feAOFbmIUkUTHW2KeEiBK6i2eLH0wANm+6WXpBNPDMzrAP7EyUd3yFu5d/x4c1tUlFliBwg1Ho8pWlu3rila+9BDwXndJUukoUPNaJHrr8/t7YZ7lfRZXOKzeDkQuuEOFb2IUkUzcqS5XLDADCX2p61bzZn97GxTuG3ECP8+PxAofOBx3qxZhVfuPXyYQnYIXbVr557kfvJJUzQrkL7/Xjr/fDOFbPBgM6y8uPc1uENxn8VtZ5/NZ/EyInTDPSp6EaWKpEULsyySZZkqmv6SkSFdfLG0Z49Z+92eywaEAl8+8HTuzAeeQPF6pVtvpXIvwtN550nDh5tjedgwszxhIPz1lwlmKSlmtMi77/KeFUqK+iyekGAu33xTmj49+O0KA46G7hUrVmjgwIFq2LChPB6P5syZU+Jjli1bphNPPFExMTE69thjNWXKlIC3E0FUkYsoVTTXXmsu33rLLMnjD3feKX31lVS9upnHTdEjhJqiPvDYNQlmzZKeeCL47aoIVq4s2MOdF4XsEOomTjSjZf7+W7r7bv8//44dUt++5sR3p05mabCYGP+/DgKrsM/i+/aZOfmWZWoDfP65060MOY6G7tTUVHXs2FEvvfSST/tv2LBB55xzjnr27Kk1a9Zo1KhRGjlypObPnx/gliKo7CJKQ4aYS86QhqdBg6Ratcz6w/Pmlf/53n9fev55sz11qtS8efmfE3BCYR949uyR/u//zP13320qAsO/KGSHcFetmjR5stl++WVp4UL/PfeBA6aHe+NGM5rt889ze0cReo7+LB4VJb3wgnTppaaj5MILAz9NIcw4Wr28f//+6t+/v8/7v/rqq2rWrJmefvppSVLbtm21atUqPfvss+rXr1+gmgkgEGJizBC3Z54xc83OPbfsz/Xbb7nzxO+7Txo40D9tBJxSWAX3u++WDh406+3ecIP5QHvZZY40LyxRyA4VQa9e0k03melXV10l/fKLGR1WHkeOmDncP/0k1a9v6rXUq+eX5sJFIiKkt982J1jmzTOf25Yvlzp0cLplISGklgz76quv1Lt373y39evXT6NGjSryMenp6UpPT8+5npycLEnKzMxUZmZmkY+z7ytuHyCQKsQxOGyYop95RtZnnylr0ybfl6vI69AhRSUlyZOaquyePeV98EEpnH9mQVIhjr9QNHasIg4cUOQrr8i68kp5K1eWFaYnmYJ9DHo2bVKkpKLKPVkej9SokbJOO433mAogrN8DH35YUfPmybNunbJvvlne8tRWycpS5KWXKmLFClkJCcr69FMzhD0cf25B5spj0OOR3n9fkf37K+Krr2T17ausZcvM6IYKytffT0iF7p07d6reUWfO6tWrp+TkZB05ckRVCpm/+dhjj2ncuHEFbl+wYIFiY2NLfM2F/hx6A5RBuB+DZ7Ztq1q//65199+vtRddVLoHW5ZOevppJf75p47UrKllw4Ypg+kmfhXux19I6tNHJ/7+uxovWybPpZfq6wce0N6OHZ1uVcAE4xg8ZtEinfDSS/JIssuo5Q3fliRZlr4dOlQ7eI+pUML1PbDGyJHqet99inj3XX2bmKidp51W+iexLJ3w0ktqsmiRvNHR+uruu7Vv2zYzbQx+48ZjMPrGG3XG9u2qtmmTMnr00KrHHlNazZpON8sRhw8f9mm/kArdZXHvvfdq9OjROdeTk5PVuHFj9e3bVwnFzDXJzMzUwoUL1adPH0VHRwejqUA+FeUY9OzbJ40YoTZffqljJ00yw5d8FPHii4pctUpWVJSiZ89W7y5dAtjSiqWiHH8h6+yzlX3ppYr85BOd/sQT8s6bJ+vUU51ulV8F6xiMeOklRf670oH32mtlnXWWIm+/PX9wSEyU9+mn1emCC9QpYC2Bm4T9e+CAAcret0+RTz6pUyZNUtZNN5m1vEsh4sEHFblokayICFnTpunU888PUGMrJtcfg127yurZU3Hr16vv008ra/FiqQIGb3sUdUlCKnTXr19fu3btynfbrl27lJCQUGgvtyTFxMQoppDKidHR0T4dwL7uBwRK2B+Dl14qjR4tz4YNil6xQurTx7fHffmldNddkiTP008rqlu3ADay4gr74y9URUdLH34onXuuPIsWKWrgwLCdWxfQY/Dxx6V77jHbo0cr8qmnzPDJwYNNlfIdO6QGDeTp2lVRFPWskML6PfDhh6V58+T5+WdF33KLWTnB1/W0n3vO/P1I8rz2mqIGDw5gQys21x6DxxxjivGdeaY8v/6q6AsuMNfj4pxuWVD5+rsJqXW6u3TposWLF+e7beHChepC7xYQumJjpcsvN9tvvOHbY3bvli66yFTQvOQS6eabA9c+wK1iYsySPF26mMI2ffuaNXJRMsuSxozJDdxjxkh24JZYRQMVQ0yMWe0jOtosR/jee749bto0s2a9JD3ySG4hU1Q8zZtL8+ebYnxffWVW38jIcLpVruRo6E5JSdGaNWu0Zs0aSWZJsDVr1mjz5s2SzNDwK6+8Mmf/66+/XuvXr9ddd92lP/74Qy+//LI+/PBD3XbbbU40H4C/XHONuZwzxwTq4mRlmd7x7dultm2lN9/0/cw8EG7i4qS5c6WOHaVdu6TevaV//4eiCJYl3XGH6eWTzFJs48bxPoKK6YQTpIceMts33VT8WvWSCVjDhpntW26R7r03oM1DCGjf3vwfio01leuvuELyep1ules4GrpXr16tTp06qVMnM0Nq9OjR6tSpk8aMGSNJ2rFjR04Al6RmzZrps88+08KFC9WxY0c9/fTTevPNN1kuDAh1HTtKnTubaqdvv138vmPGmHWL4+Kkjz6S4uOD00bArapXNx90WrUygbtPHxPAUVB2tllu7ZlnzPUXXjBLsQEV2d13S6ecYpYkHDnSnJgqzNdfm/WZs7LMKJBnn+VkFYwuXcxoCXvq0403Fn0cVVCOhu4ePXrIsqwCX1OmTJEkTZkyRcuWLSvwmB9++EHp6en6+++/NXz48KC3G0AA2L3db75Z9Bv1J5+YNYoladIk09MNwBRAWrTIzLFbu1bq10/65x+nW+UuWVlmXeJXXzVBYdIk07MHVHRRUeaEd+XKpif7tdcK7vPHH9I550ipqWYqy5QppSp8igqgXz/pnXfM++trr0kPPOB0i1yFvxYA7nDppab3eu1aacWKgvf//bdkTze59VYzlxtArsaNTfCuV0/68UfzATklxelWuUNGhnTZZWb+amSkmbt69dVOtwpwjzZtzFQLyUy/WLtWWrZMmj5dmjHDjKDZt8/0iH/0kVSpkqPNhUtdcon0yitm+9FHc0cVgdANwCWqVjXD1aSCBdWOHDHVhA8elE4/XXriieC3DwgFLVuaoeZ2UZsLLpDS0pxulbPS0syQ2BkzTFCYOTP3vQZArptvNoUDU1PNPN2ePc3JqosvNnO9GzaUPvuMaV0o3nXXmQJ7knT77dLkyc62xyUI3QDcwx5iPmOGGUo+fbo5037DDdKaNVKdOmauEGfYgaJ16CB9/rkZObJokQmYWVlOt8oZqanSwIHSf/9rhs5+8ok0aJDTrQLcKSIidxRZYRWod+wofCQacLR775VGjzbbI0eaQrkVHKEbgHt07iw1aWL+2Z9/vjnD3rOnmTvm8ZgQ3qiR060E3O+000zAtJcVu/pqU0SsIklOls4+25x4iI+X5s0zcw4BFM7rze2hLMqoUVSmRsk8HrMM4/Dh5n/PJZeYIrgVGKEbgHvMni1t2lT4fZZlhpcD8M1ZZ5mRIZGRprjNzTdXnGqy+/dLvXpJq1aZofYLF0rduzvdKsDdVq4sfskwy5K2bDH7ASXxeMx0wUGDTGfKeedJq1c73SrHELoBuIPXawqkFcXj4Qw7UFrnnWeKh3k80ssvS/ff73SLAm/XLjMvdfVqqXZtackS0/MPoHg7dvh3PyAqyoxS7NnTFPbs399Uwq+ACN0A3IEz7EBgXHZZbjXZxx6THn/c2fYE0tatpkf755+lBg2k5culTp2cbhUQGho08O9+gGTqacyZI510krR3r6mEv3mz060KOkI3AHfgDDsQONddlxu277knN4SHkw0bpG7dpD//NOuVr1ghtWvndKuA0NG1q5SYaEbGFMbjMUsTdu0a3HYh9CUkmAKfrVubk6N9+0p79jjdqqAidANwB86wA4F1113SffeZ7RtvlN5919n2+NOff5ogsGGDdOyxJnAfe6zTrQJCS2Sk9NxzZvvo4G1fnzjR7AeUVp06pr5G48bmPfvss03BywqC0A3AHTjDDgTehAnSTTeZ6RrDh0sff+x0i8rvp59MD/e2baZne8UKswoCgNJLSjJr2R+9Ukhiork9KcmZdiE8NG5sgnft2tL335uVatLSnG5VUBC6AbgDZ9iBwPN4zN/ZlVeaooQXXywtXux0q8pu9WpTNG33bjN3e9kyRsMA5ZWUJG3caJZ4mjbNXG7YQOCGf7RubZZwrFrVvGdfcomUleV0qwKO0A3APTjDDgReRIQ0aZJ0wQVmGZfzz5e++srpVpXeqlVmWbR//jHVyZcsMcMXAZRfZKQ5oTVkiLnkhDf86aSTpE8+kWJizOXIkWY97zBG6AbgLpxhBwLPXsalTx8pNVUaMED68UenW+W7RYukfv2kQ4dMIFiwwKzHDQAIDT16SB98YE7ovP22dMcdZupTmCJ0A3AfzrADgRcTI82eLZ1+unTggKkmu3at060q2WefSeeeKx0+bArxfPaZGaYIAAgt559vRl5J0rPPSo8+6mx7AojQDQBARRUXZ0LrCSeYedG9e5v1U71eM9du+nRz6fU63NB/zZghDRokpaeb4fFz5kixsU63CgBQVsOGSc88Y7YfeMAsaenW/0HlEOV0AwAAgIOqV5fmz89d4/q008ztO3bk7pOYaAqwOTnNY+pU6aqrzLy/IUPMcMToaOfaAwDwj9tuk/btkx55RLrhBhO+9+/Pvd8N/4PKiZ5uAAAqurp1zTIudeqYsJ03cEtmOa7Bg6VZs4LTHq9XnuXL1WjFCnmWL5deftn0hmRnSyNGSO+8Q+AGgHDy8MNmmpOUP3BLwf8fFACEbgAAIDVsWHT9BLu4zahRgR/mN2uW1LSpovr00cnPPKOoPn2kG2809918s/T669R5AIBwk50t/fZb4fcF839QgDC8HAAASCtXSjt3Fn2/ZUlbtkjNmkn160sJCaaAWUJC7lfe60VtV6li1gsvzKxZpjejqAq23bubJc8AAOFl5Upp69ai77f/B61caYrshhhCNwAAKDikvChbtpivsoqMLDyQV60qzZ1bdOD2eMy8v0GD6OkGgHDj6/8gX/dzGUI3AACQGjTwbb9nn5VatDBrZCcnm6+itvNeP3TIBGqv1yxRduBA6doX4r0cAIBi+Po/yNf9XIbQDQAApK5dTYXYbdsK7232eMz9N99ctp7m7GyztnZR4XzpUlORvCQh2ssBACiGr/+DunYNftv8gNANAABMkH7uOTOn2uPJ/6HHnoM9cWLZh3ZHREjx8earYcOC9zdp4lvoDtFeDgBAMQL9P8hhVCMBAABGUpI0c6bUqFH+2xMTze2BXCPV7uUoqsiaxyM1bhyyvRwAgBI4+T8owOjpBgAAuZKSpPPPN3Ond+wwPctduwa+dyHMezkAAD5w6n9QgBG6AQBAfpGRzhQrs3s5br01/9IxiYkmcIdwLwcAwEdO/Q8KIEI3AABwj397ObKWLtWazz/XCf37K6pnz5Dv5QAAVFyEbhfyesNuRAUAAL6LjJTVvbu2paaqY/fu/BMEAIQ0QrfLzJpV+Ki6554L3qg6Qj8AAAAA+AfVy11k1ixTPyZv4JbMcnWDB5v7g9GGpk2lnj2lyy4zl02bBue1AQAAACDcELpdwus1PdyFrQVv3zZqlNkvUNwQ+t3C65WWLZOmTzeXgfy5AwAAAAhfDC93iZUrC4bdvCxL2rJFql1bqllTio+X4uLMpf2V97qv27GxZiWWkkK/x2NC//nnh/9QczcM8UfFxhQPAACA8EHodokdO3zb78AB8+UvHo8J39HR0j//FL2fHfo//lgaNEiKCOAYCScDh93bf/TJB7u3f+bM4ARvr1davtyjFSsaKS7OIwr3Vhyc9AEAAAgvhG6XaNDAt/3efFNq105KScn9Sk0t/XZqqnk+yzK3+erCC6VKlUwIaNxYOuaY/Jf2drVqpf8ZSM4FDssyJzNuvLH43v5bbw18b3/uzyBK0sl65hmK6VUUbjnpAwAAAP8hdLtE164mWG3bVnjo83jM/cOH+yf8ZGdLhw/nhvGlS6VrrvHtsRkZ0vr15qsoVasWDOJ5txMTpcqV8z/GX4EjLU3at8987d+fu3309aO3s7KKf17LMicD4uOlOnWkGjVK/xUdXfxruCF00dPqzEkHpngAAACEJ0K3S0RGmlAzeLD5cJ33g7fHYy4nTvTfh+2IiNy53fXqmQrl48aVHPrXrpV27ZI2bzbDzbdsyd22L/fvlw4dkn791XwVpW7d3CCemCi9807xheSuu870Rv/zT9Hhed8+6cgRf/yEipaWlvu9l1ZcXNGBvFo18zsuqaf9vPOkqAD95boh9EvOTzHw90mHtDRp715pzx7zZW/nvfzrL9/qOtxzjzRggNSihdSoUWB+LkxvAAAA8B9Ct4skJZlQU9gH/okTAxt2fA39lStLTZqYr6KkphYM5EdvHz4s7d5tvr77zrc27t0rjRjh+/dTs6ZUq5b5yrtd1PVff5XOPrvk5373XalVKxP+ff06eDD3Z5OaWny4Kord0x4Tk1sEL+9XlSoFbyvsq6j9YmKkm25yvqfVyZ52X046DBpkfp+FheeiAnVppnCU5KmnzJdkpno0a2YC+NFfzZoVHE3iCzdMbwCc5vQUG6dfHwDgX4Rul0lKMqHGiX+2/gr9cXFSmzbmqzCWZXqn8wbx+fOlTz8t+bk7dJCOP77w8Jx3OyEh92SBrxo08G2I/6WXlv734fWaoFZYIN+/31yuXm2G+ZckO9uMJDh0qHRtKC+7p/WCC0xdgaN76qtXz99rX5ZjNhg97ZZlTnwcOiQlJ+f+LP/5x0yxKG60xUUXmcvs7NK/blSUWX2gTp3cy7zbu3ZJDz9c8vN07mxGfGzcaKZ6/Pmn+Tqax2N6wps3LzyU16xZ8DFuGekAOMnpKTZOvz4AwP88llXYR8zwlZycrGrVqungwYNKSEgocr/MzEzNnTtXAwYMUHRJE3HDjBNn2Jctk3r2LHm/pUulHj0C1w47dEiF9/YHMnT4+jOYMUM64QQzWqCwryNHir6vuP2Sk6X0dP99PwkJxQfzo68nJEi9exddyd/jMcfj/Pm57bUDc2HbRd1/6FDhwbq0qlbNH5qLC9S1a5sTEcWdCPJ6zTSPkk76bNhg/h69XnMS5O+/C/8q6aRM9eoFe8bvv9/0zhfm6NcHAs2J/8NFnXgKxv8AN7y+W7ihp78ifw6EO3AMhgZfsyU93SggMjKwwbYwvhaS69o1sO1wcoi/rz+DCy4IzIcPX0P/sGEmsP3zT+4ce/vrwIHcyvjJyeZr0yb/tM+ypO3bpfbt/fN8EREmOCckmMv0dBNWS/Lii9LIkWY4vj+Vtq5DZKQJ6U2bSr165X8uyzLD2osK5Dt3mt/Vd9/5Pr3DHukwc6ZZxSBQdQUkd3zgdho/g+ArqZihZEbDpKaaopgREeZ3EhHh23ZJ90slr6BREYopuqGnn7oWvAcB/kZPdxE4uxR8TvYyH82pfzZO/gxK29NalIyM/GG8sGBe2PVdu3wrghcXZ3qO8wZmX7aPvq1Klfw9z24abXH0B87Gjf170ic11aw+kDeIr1ol/fyzb4+PiJAaNjQrERT1Vb166ad4SO75wO3kh003/Ayc5vVKS5dm6fPP16h//xPUs2eU334HWVmFjxL58UffTrw5bcoUaejQwJ74coobevr5++Nn4BZkkdDga7YkdBeBA90ZwQgcbufkzyAUhtcHKvT666SDv9ri1ike9tD2ksTHFx/KGzUyheDy4gO3O34GUuivIJCSknti6egTTJs2lbxEZHGOO86svpGdbX5O2dnl27Yv09LM1BlfxMSY2hodOpjRP+3bm+169cp2sssN7Pfg4gqN1q0rff65OXEaF5f75a9j0y1/f5LzJ//d8DNwktMnXyWySKggdBeB0O1+bnijc1oge3lK4lTod0PoddNoi2Dz9ef/999m6PrmzUV/7d1b8uvZc/QbNzYhPDFRmjzZjHwoav9g/f4D/WEzO9uMCElPz/91+LDUp49Z1aEwwTrx48YVBI7+HViW+Tkd3VttB+xdu4p/nZiY/JX/mzc3P//77iu5jYE68efria+YmKLrb9SunRvE7cvjjjMrVJRGoP4PW1ZuIUj7a9Mmc/nLL2UfaWCv6lHUV2xs8ffHxZnVHi691Pm/P8m5v8GSTnxUlNoeTp98tZFFQgOhuwiEboQKJ49Bp8+wS86F3oo82sJfP//Dh83Pr7hgXtaifXXrml706GgzvDbvZXlui4iQnn02d3m/wiQkSNdeW3hgLs1XZmbZvndbrVpS/fq+FSjMe1tsbMm9oE72cvnS01mlignJGzbk1o8oSo0ahVfub97cjLSw51Ef/fpOnfgrzYmvzZvNdJCffsq9XLeu8JUVPB7p2GPzB/EOHczP4eifgVS+wGGvTpI3TB8dsJOTff2JFC4hwXyfqan+KYpZWscdZ34eR09b8uV6XFxw/gYzM81oj7xfqakFbzv6a8MGacWKkn8GgZ5m5SQ39fSTRUIDobsIhG6Eiop6DLoh9Fbk0RbB+PlblqmSbgfwLVukBQukuXP98/yhplKl3MJ8gVwKMDq6+GCekCD93/+ZGgtFqV3bFBPMzCz+5ENJJyYKuz8lpXSBzOMxx2Zhy+I1b26+p9Jy+sRfeV7/yBHpt9/yB/Gffy665zY21gTIvEF80ybp6quLDhwz/r+9e4+NqszDOP5MS1tKaYFS2s4ALZdiKSCogLVR8AKRwi5ZbisoMYUQCVqIgngjYmE1MXE3SrxhYhT+kKJiBNF4iSJgZAsa3XJb6ELjBrWUm0tbikhDz/5xMqXTzjCl9PSdy/eTnMxw5rT9nZl3DvPMOe/7bpJuv93/mWrvcu5c8P3MyJCysy8PBjlggP3aP/lk8J/1Bj7Lsi/Jr6/3Xc6fb72uLdscPx549oyOEhNjf2kYKJR37y69886VjwNJSdKf/3x5H/yF54sXnd2Pnj2lMWPs9jNihH07bJg9S0c4C7Uz/dH6OTDcELoDIHQjXERzG4zm0BsKTHRvaOultW+8YYeDhga7X66/2/Y8duiQ9PXXwf/+lCnSqFF2SL7WpWtXO3B7A83VPAc5OVceoLDlurb0ww8Xjz0mLVhgfzju6FkEJPNf/HX03z9xwg7fzYP4wYN2YL1aLWdWCCQz83KYbhmus7L8X+5u+kqDtr7//vY3e59aTkUZ7N/+rkJwWny8HeSbL0lJrdd5l6oq+4qG9urXzw7gzZdhw+wvFK6Wk10camrsL4uOHbNvvff377f/Lwjm3nvtWUNycuwv+Twe/1eNXAuT3QxDQTh9DiR0B0DoRrigDcKkzm5/4fKB28nLKp16DizLPvsVbCaBf/1L2rUr+O/LzbVDYELC5bP0gZareXzvXvssazCdcWmr6Q98Tv/9S5fsy9GbB/E9e9p+ptfjaR2mvQE7K8vuBtAekTCDhz+WZZ+ZDhTKvffLyqTNm4P/vvvvl8aPv3KYTkpqPVhlMG15Djwe6b337HB68ODlpaoq8O/NymodxvPy7Dr9uZYuDpcu2e24eahuedvRVxR17Xr5KhtvEM/JsZesrKufaSBU+pSbEm77T+gOgNCNcEEbhEkm2l+kfuC+GswgYP41iFYbN0r33Rd8u/XrpaIi5+qI1hk8JPPvQan9z8H//md3b2gexA8elKqrA/+tAQNah/H//MeeEi9QF4cNG6Qbb2x9ltp7+8svbZudIC3N/pLI+0WR9+qFkpLgPzttmt2do7LSPh5d6UqiLl3s/WwZxgcPtgdz7NrVd/tQ6lNuQjjuP6E7AEI3wgVtECaZan/R/IG7eR3MIBB9MwiYFgqBzyvUpqzrrGNQKLwHpY59Dn77rXUQP3gw8FgDHaFLF/t58obp5sHaO21lR3VxaGiww35lpX31yNGjl+9XVl550FDvuBTeMD5okPSPf0hnzgTePpK/eAy1PvVtRegOgNCNcEEbhEnROHq+ZL4/rxczCJh/DaJNqAS+UGB62k7T70HJ+WPQ6dOXA/iBA/ZteXnbBlP0Xs7dPEw3v+92t7/Wjnz+Gxvty+6bB/Hm99t7mfsnn0h/+lP7fjaUnDljXx3hXb79Vvrxx+A/F2qj5xO6AyB0I1zQBmFSNLc/0/15TQuF0BvtgwiZEiqBLxSYPAaGwnvQhNJS+9LyYDZsaFtXiPbqzFk8mofxbdvaNq6GZE+fmZMjDRnS+vYK8abNOur/Qcuyuxj8+9/2OADNQ/apU+2rrbTUHswuVLQ1W15l134AACJbbGxofYve2WbMkP7yF7NfPMTGSrffbqm+/lfdfvsoAncnmTHDDtb+BjGK9MAXSkLhPWiCx9Ox27VXZzz/LpcdnNPTpYICe92dd7ati4dkX55/8qT0z3+2fqxPH/9hPCenbdO6tWcgM8uyp//0BurmAfvs2cB/a8AAe4T7vDx7BPi//z14fW538G1CEaEbAAD4iPYvHqJZtAa+UBON78Fx4+xwF6yLw7hxztdi4vlv6/6Xl0v//a905Ih9hrz57cmT9hnkU6cCB3J/YTwnx55/PdBAZr/+aq9//33phhtan7U+fNieJcOfmBi7O8CwYZeXvDxp6FB7lH2vS5fsAR1D4fV3AqEbAAAATaIx8MG82Fj7bOqsWa3nhPd2cVizJnK/AGrr/qem2stNN7X+HbW19iXr/gL5iROXA3lZWeuf7d3b7mfuL/B61/31r4Hrj4uzQ3zzcD1smL2u5Sjt17L/4fr6E7oBAAAAGBftXRyudf9TUuwp1W68sfVjdXWX+4+3DOXV1YFHTW8pLs6e3s17xtobrgcPth+7FpH8+hO6AQAAAISEaO/i4N3/jh5MMjn5yoH81VelFSuC/55169o24F17RerrT+gGAAAAEDKivYtDZw8mmZx8eUC3YPr2dbYWKTJf/xjTBQAAAAAAzPEO5ObtP92Sy2VPnRauA5mZRugGAAAAgCjmHchMah28I2EgM9MI3QAAAAAQ5bwDmbW8hLxfP3t9OA9kZhp9ugEAAAAAETuQmWmEbgAAAACApMgcyMw0Li8HAAAAAMAhhG4AAAAAABxC6AYAAAAAwCGEbgAAAAAAHELoBgAAAADAIYRuAAAAAAAcQugGAAAAAMAhhG4AAAAAABxC6AYAAAAAwCGEbgAAAAAAHELoBgAAAADAIYRuAAAAAAAcQugGAAAAAMAhhG4AAAAAABxC6AYAAAAAwCFdTBfQ2SzLkiTV1tZecbuGhgadP39etbW1iouL64zSAB+0QZhE+4NptEGYRPuDabTB8ODNlN6MGUjUhe66ujpJUv/+/Q1XAgAAAAAId3V1derRo0fAx11WsFgeYRobG1VVVaXk5GS5XK6A29XW1qp///76+eeflZKS0okVAjbaIEyi/cE02iBMov3BNNpgeLAsS3V1dfJ4PIqJCdxzO+rOdMfExKhfv35t3j4lJYWGDqNogzCJ9gfTaIMwifYH02iDoe9KZ7i9GEgNAAAAAACHELoBAAAAAHAIoTuAhIQElZSUKCEhwXQpiFK0QZhE+4NptEGYRPuDabTByBJ1A6kBAAAAANBZONMNAAAAAIBDCN0AAAAAADiE0A0AAAAAgEMI3QG89tprGjBggLp27ar8/Hx99913pktCFFi1apVcLpfPMnToUNNlIYJ98803mjp1qjwej1wul7Zs2eLzuGVZeuaZZ+R2u5WYmKiJEyfqyJEjZopFRArWBufNm9fquFhYWGimWESc559/XmPHjlVycrLS09M1bdo0VVRU+Gxz4cIFFRcXq3fv3urevbtmzpypEydOGKoYkaQt7e+OO+5odQxctGiRoYrRXoRuP9577z0tW7ZMJSUl+vHHHzVq1ChNmjRJJ0+eNF0aosDw4cN1/PjxpuXbb781XRIiWH19vUaNGqXXXnvN7+MvvPCCXn75Zb3xxhvas2ePkpKSNGnSJF24cKGTK0WkCtYGJamwsNDnuLhx48ZOrBCRbOfOnSouLtbu3bv15ZdfqqGhQXfffbfq6+ubtlm6dKk+/vhjbdq0STt37lRVVZVmzJhhsGpEira0P0l64IEHfI6BL7zwgqGK0V6MXu5Hfn6+xo4dq1dffVWS1NjYqP79+2vJkiV68sknDVeHSLZq1Spt2bJF5eXlpktBFHK5XNq8ebOmTZsmyT7L7fF49Oijj2r58uWSpJqaGmVkZGj9+vWaM2eOwWoRiVq2Qck+03327NlWZ8ABJ5w6dUrp6enauXOnxo8fr5qaGvXp00elpaWaNWuWJOnw4cPKy8tTWVmZbrnlFsMVI5K0bH+Sfab7hhtu0Jo1a8wWh2vCme4WLl68qB9++EETJ05sWhcTE6OJEyeqrKzMYGWIFkeOHJHH49GgQYM0d+5cHTt2zHRJiFI//fSTqqurfY6HPXr0UH5+PsdDdKodO3YoPT1dubm5evDBB3XmzBnTJSFC1dTUSJJSU1MlST/88IMaGhp8joNDhw5VVlYWx0F0uJbtz2vDhg1KS0vTiBEj9NRTT+n8+fMmysM16GK6gFBz+vRpXbp0SRkZGT7rMzIydPjwYUNVIVrk5+dr/fr1ys3N1fHjx7V69WqNGzdOBw4cUHJysunyEGWqq6slye/x0PsY4LTCwkLNmDFDAwcOVGVlpVasWKHJkyerrKxMsbGxpstDBGlsbNQjjzyiW2+9VSNGjJBkHwfj4+PVs2dPn205DqKj+Wt/knTfffcpOztbHo9H+/bt0xNPPKGKigp9+OGHBqvF1SJ0AyFk8uTJTfdHjhyp/Px8ZWdn6/3339eCBQsMVgYAZjTvxnD99ddr5MiRGjx4sHbs2KEJEyYYrAyRpri4WAcOHGAsFRgRqP0tXLiw6f71118vt9utCRMmqLKyUoMHD+7sMtFOXF7eQlpammJjY1uNSnnixAllZmYaqgrRqmfPnrruuut09OhR06UgCnmPeRwPEUoGDRqktLQ0jovoUIsXL9Ynn3yi7du3q1+/fk3rMzMzdfHiRZ09e9Zne46D6EiB2p8/+fn5ksQxMMwQuluIj4/X6NGjtW3btqZ1jY2N2rZtmwoKCgxWhmh07tw5VVZWyu12my4FUWjgwIHKzMz0OR7W1tZqz549HA9hzC+//KIzZ85wXESHsCxLixcv1ubNm/X1119r4MCBPo+PHj1acXFxPsfBiooKHTt2jOMgrlmw9uePd7BdjoHhhcvL/Vi2bJmKioo0ZswY3XzzzVqzZo3q6+s1f/5806Uhwi1fvlxTp05Vdna2qqqqVFJSotjYWN17772mS0OEOnfunM+35T/99JPKy8uVmpqqrKwsPfLII3ruuec0ZMgQDRw4UCtXrpTH4/EZXRq4Fldqg6mpqVq9erVmzpypzMxMVVZW6vHHH1dOTo4mTZpksGpEiuLiYpWWluqjjz5ScnJyUz/tHj16KDExUT169NCCBQu0bNkypaamKiUlRUuWLFFBQQEjl+OaBWt/lZWVKi0t1ZQpU9S7d2/t27dPS5cu1fjx4zVy5EjD1eOqWPDrlVdesbKysqz4+Hjr5ptvtnbv3m26JESB2bNnW26324qPj7f69u1rzZ492zp69KjpshDBtm/fbklqtRQVFVmWZVmNjY3WypUrrYyMDCshIcGaMGGCVVFRYbZoRJQrtcHz589bd999t9WnTx8rLi7Oys7Oth544AGrurradNmIEP7aniRr3bp1Tdv8/vvv1kMPPWT16tXL6tatmzV9+nTr+PHj5opGxAjW/o4dO2aNHz/eSk1NtRISEqycnBzrscces2pqaswWjqvGPN0AAAAAADiEPt0AAAAAADiE0A0AAAAAgEMI3QAAAAAAOITQDQAAAACAQwjdAAAAAAA4hNANAAAAAIBDCN0AAAAAADiE0A0AAAAAgEMI3QAAoMO4XC5t2bLFdBkAAIQMQjcAABFi3rx5crlcrZbCwkLTpQEAELW6mC4AAAB0nMLCQq1bt85nXUJCgqFqAAAAZ7oBAIggCQkJyszM9Fl69eolyb70e+3atZo8ebISExM1aNAgffDBBz4/v3//ft11111KTExU7969tXDhQp07d85nm7ffflvDhw9XQkKC3G63Fi9e7PP46dOnNX36dHXr1k1DhgzR1q1bnd1pAABCGKEbAIAosnLlSs2cOVN79+7V3LlzNWfOHB06dEiSVF9fr0mTJqlXr176/vvvtWnTJn311Vc+oXrt2rUqLi7WwoULtX//fm3dulU5OTk+f2P16tW65557tG/fPk2ZMkVz587Vb7/91qn7CQBAqHBZlmWZLgIAAFy7efPm6Z133lHXrl191q9YsUIrVqyQy+XSokWLtHbt2qbHbrnlFt100016/fXX9eabb+qJJ57Qzz//rKSkJEnSp59+qqlTp6qqqkoZGRnq27ev5s+fr+eee85vDS6XS08//bSeffZZSXaQ7969uz777DP6lgMAohJ9ugEAiCB33nmnT6iWpNTU1Kb7BQUFPo8VFBSovLxcknTo0CGNGjWqKXBL0q233qrGxkZVVFTI5XKpqqpKEyZMuGINI0eObLqflJSklJQUnTx5sr27BABAWCN0AwAQQZKSklpd7t1REhMT27RdXFycz79dLpcaGxudKAkAgJBHn24AAKLI7t27W/07Ly9PkpSXl6e9e/eqvr6+6fFdu3YpJiZGubm5Sk5O1oABA7Rt27ZOrRkAgHDGmW4AACLIH3/8oerqap91Xbp0UVpamiRp06ZNGjNmjG677TZt2LBB3333nd566y1J0ty5c1VSUqKioiKtWrVKp06d0pIlS3T//fcrIyNDkrRq1SotWrRI6enpmjx5surq6rRr1y4tWbKkc3cUAIAwQegGACCCfP7553K73T7rcnNzdfjwYUn2yOLvvvuuHnroIbndbm3cuFHDhg2TJHXr1k1ffPGFHn74YY0dO1bdunXTzJkz9eKLLzb9rqKiIl24cEEvvfSSli9frrS0NM2aNavzdhAAgDDD6OUAAEQJl8ulzZs3a9q0aaZLAQAgatCnGwAAAAAAhxC6AQAAAABwCH26AQCIEvQoAwCg83GmGwAAAAAAhxC6AQAAAABwCKEbAAAAAACHELoBAAAAAHAIoRsAAAAAAIcQugEAAAAAcAihGwAAAAAAhxC6AQAAAABwCKEbAAAAAACH/B/hD9PyqLEZ+wAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 1000x600 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Load data from CSV into a DataFrame\n",
        "file_path = save_progress_path  # Replace with your CSV file path\n",
        "df = pd.read_csv(file_path)\n",
        "\n",
        "# Plotting the line plot\n",
        "plt.figure(figsize=(10, 6))\n",
        "plt.plot(df['epoch'], df['train_loss'], marker='o', linestyle='-', color='b', label='Train Loss')\n",
        "plt.plot(df['epoch'], df['val_loss'], marker='o', linestyle='-', color='r', label='Validation Loss')\n",
        "\n",
        "plt.title('Training and Validation Loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.legend()\n",
        "plt.grid(True)\n",
        "plt.tight_layout()\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iOQs9Sy2VBHR"
      },
      "source": [
        "# Sample Predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RnM0qtplSwyW"
      },
      "outputs": [],
      "source": [
        "model.to(DEVICE)\n",
        "model.eval()\n",
        "idx = 1995\n",
        "mask = val_dataset[idx][\"mask\"].unsqueeze(0)\n",
        "print(\"Mask Shape: \",mask.shape)\n",
        "image = val_dataset[idx][\"image\"].unsqueeze(0)\n",
        "print(\"Image Shape: \",image.shape)\n",
        "raw_output = model(image.to(DEVICE))\n",
        "print(\"raw output shape: \", raw_output.shape)\n",
        "print(\"------ Raw Output ------\")\n",
        "print(raw_output)\n",
        "print(\"------- Pred Probs -------\")\n",
        "pred_probs = torch.sigmoid(raw_output)\n",
        "print(pred_probs)\n",
        "print(\"------- How does the Mask look like? ------\")\n",
        "print(mask)\n",
        "print(\"-------- Predicted Segmentation mask --------\")\n",
        "# binarizer_fn = TripletMaskBinarization(triplets=[[0.7, 600, 0.3]])\n",
        "# mask = binarizer_fn.transform(mask).float()\n",
        "# print(segmentation_mask)\n",
        "segmentation_mask = (pred_probs > 0.4).float()\n",
        "print(segmentation_mask)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cALrPDiZFyhu"
      },
      "outputs": [],
      "source": [
        "mask.unique()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "prdvPG3XZiuL"
      },
      "outputs": [],
      "source": [
        "segmentation_mask.unique()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iAzD4mVaTYJg"
      },
      "outputs": [],
      "source": [
        "plt.style.use(\"classic\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ak3eAHrDdY_V"
      },
      "outputs": [],
      "source": [
        "first_channel_tensor = image[0, 0, :, :]\n",
        "print(first_channel_tensor.shape)  # torch.Size([512, 512])\n",
        "second_channel_tensor = image[0, 1, :, :]\n",
        "print(second_channel_tensor.shape)  # torch.Size([512, 512])\n",
        "third_channel_tensor = image[0, 2, :, :]\n",
        "print(third_channel_tensor.shape)  # torch.Size([512, 512])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LRJzqWT1drZv"
      },
      "outputs": [],
      "source": [
        "plt.title(\"First Channel Image\")\n",
        "plt.imshow(first_channel_tensor.detach().cpu().numpy(), cmap=\"gray\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tS4Lttr2TLad"
      },
      "outputs": [],
      "source": [
        "plt.imshow(mask.squeeze().detach().cpu().numpy(), cmap = 'gray')\n",
        "print(mask.squeeze().shape)\n",
        "plt.title(\"Mask\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_X9nSoL1TNw_"
      },
      "outputs": [],
      "source": [
        "print(segmentation_mask.squeeze().shape)\n",
        "plt.imshow(segmentation_mask.squeeze().detach().cpu().numpy(), cmap='gray')\n",
        "plt.title(\"Segmentation Mask\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xi-kZiELUV79"
      },
      "outputs": [],
      "source": [
        "print(type(pred_probs), type(mask))\n",
        "print(pred_probs.shape, mask.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nKAY7_pFh_Vt"
      },
      "outputs": [],
      "source": [
        "print(segmentation_mask.squeeze().squeeze().shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Sm9Qm1_FiFbS"
      },
      "outputs": [],
      "source": [
        "print(mask.squeeze().squeeze().shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7kiIFEw2TPZn"
      },
      "outputs": [],
      "source": [
        "# \"\"\"\n",
        "# Both methods aim to capture the overlap between predicted positive regions and actual positive regions in the ground truth.\n",
        "# The dice_metric approach leverages the full range of predicted probabilities, while the  metric function relies on a binary classification based on a chosen threshold.\n",
        "# \"\"\"\n",
        "# Dice = metric(pred_probs.detach().cpu(), mask).item()\n",
        "# print(f\"Dice coefficient: {Dice}\")\n",
        "# dice_metric_score = dice_metric(pred_probs.detach().cpu(), mask)\n",
        "# print(f\"Dice coefficient: {dice_metric_score}\")\n",
        "\n",
        "\"\"\"\n",
        "Both methods aim to capture the overlap between predicted positive regions and actual positive regions in the ground truth.\n",
        "The dice_metric approach leverages the full range of predicted probabilities, while the  metric function relies on a binary classification based on a chosen threshold.\n",
        "\"\"\"\n",
        "Dice = metric(segmentation_mask.detach().cpu(), mask).item()\n",
        "print(f\"Dice coefficient: {Dice}\")\n",
        "dice_metric_score = dice_metric(segmentation_mask.squeeze().squeeze().detach().cpu(), mask.squeeze().squeeze(), per_image=False)\n",
        "print(f\"Dice coefficient: {dice_metric_score}\")"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "Y7HFiESzDWcY",
        "3A4XlffPf1jV",
        "7xtZrBIJeLH_",
        "A-y8ovXEK2St",
        "od_pZDUoK2Su",
        "BNLEHf3QK2Sv",
        "6-dcysMmK2Sw",
        "Vzze2VhIK2S5",
        "kOlM9jTEK2S5",
        "K136b3eqK2S5",
        "pOkP2HYpvsqr",
        "LcACWXgAXHQu",
        "gJFz9wAhzL4k",
        "Ay8rDOynkDJU",
        "v3bIua6-VHh9"
      ],
      "gpuType": "A100",
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.9"
    },
    "toc": {
      "base_numbering": 1,
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": false,
      "toc_position": {
        "height": "calc(100% - 180px)",
        "left": "10px",
        "top": "150px",
        "width": "165px"
      },
      "toc_section_display": true,
      "toc_window_display": false
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}