hierarchical_FCOS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "25988e6a",
   "metadata": {},
   "source": [
    "## Hierarchical Anchor-free Object Detection for Atypical and Typical Mitosis Subtyping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d97bcda9-1b3b-44cd-9948-a2eb0d5d89f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "run = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b5250031-684e-4dd3-be07-9bb171588349",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import pickle\n",
    "import numpy\n",
    "from SlideRunner_dataAccess.database import Database\n",
    "import pytorch_lightning as pl\n",
    "import albumentations as A\n",
    "import wandb\n",
    "\n",
    "from src.datamodule import MitosisTwoSubclassDataModule\n",
    "from src.fcos.litmodel import LitHFCOS\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85cd248a",
   "metadata": {},
   "source": [
    "## Build annotations dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a55be88b",
   "metadata": {},
   "outputs": [],
   "source": [
    "db = 'databases_atypia/MIDOG.sqlite'\n",
    "DB = Database().open(db)\n",
    "classes = {x:y for x,y,_,_ in DB.getAllClasses() }\n",
    "databases_directories = [['databases_atypia/MIDOG.sqlite', 'images_midog_training/'],\n",
    "                         ['databases_atypia/MIDOG_preliminaryTest.sqlite', 'images_midog_preliminarytest/'],\n",
    "                         ['databases_atypia/MIDOG_final_TestSet.sqlite', 'images_midog_finaltest/']]\n",
    "\n",
    "annotations_dict={}\n",
    "class_mapping = { 'mitotic figure' : -1,\n",
    "                  'Mitosis - AMF OTHER': 0,\n",
    "                  'Mitosis - AMF BIPOLAR ASYMMETRIC CAN ALSO HAVE SEGREGATION': 0,\n",
    "                  'Mitosis - AMF MULTIPOLAR ASYMMETRY CAN ALSO HAVE SEGREGATION': 0,\n",
    "                  'Mitosis - AMF SEGREGATION ONLY':0,\n",
    "                  'Mitosis - RING SHAPE (PROMETAPHASE OR METAPHASE)': 1,\n",
    "                  'Mitosis - NORMAL ANAPHASE-TELOPHASE': 2,\n",
    "                  'Mitosis - NORMAL PROMETAPHASE': 3,\n",
    "                  'Mitosis - NORMAL METAPHASE': 4\n",
    "                }\n",
    "\n",
    "subclass_names = ['N/A','AMF', 'RING SHAPE', 'ANAPHASE-TELOPHASE','PROMETAPHASE','METAPHASE']\n",
    "value_to_class_DB = {y:x for x,y,_,_ in DB.getAllClasses() }\n",
    "value_to_class = {x:y for x,y in enumerate(subclass_names) }\n",
    "\n",
    "\n",
    "for dbfile, folder in databases_directories:\n",
    "    DB.open(dbfile)\n",
    "    for uid, fname in DB.listOfSlides()[0:150]:\n",
    "        fname = folder+fname\n",
    "        DB.loadIntoMemory(uid)\n",
    "        annotations_dict[fname] = {}\n",
    "        for annoI in DB.annotations:\n",
    "            anno = DB.annotations[annoI]\n",
    "            if (value_to_class_DB[anno.agreedClass] not in class_mapping): # skip non-mitotic figures\n",
    "                cls = -2\n",
    "            else:\n",
    "                cls = class_mapping[value_to_class_DB[anno.agreedClass]]\n",
    "            annotations_dict[fname][annoI] = {'x': [int(anno.x1-25),int(anno.x1+25)], 'y': [int(anno.y1-25), int(anno.y1+25)], 'class': cls}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8f2597a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_values([{'x': [819, 869], 'y': [196, 246], 'class': -2}, {'x': [2354, 2404], 'y': [204, 254], 'class': 4}, {'x': [2207, 2257], 'y': [480, 530], 'class': 3}, {'x': [2688, 2738], 'y': [321, 371], 'class': -2}, {'x': [1603, 1653], 'y': [73, 123], 'class': -2}, {'x': [4997, 5047], 'y': [136, 186], 'class': -1}, {'x': [511, 561], 'y': [1751, 1801], 'class': 4}, {'x': [2309, 2359], 'y': [1100, 1150], 'class': -2}, {'x': [2598, 2648], 'y': [1710, 1760], 'class': -2}, {'x': [4302, 4352], 'y': [1715, 1765], 'class': -2}, {'x': [417, 467], 'y': [2314, 2364], 'class': 3}, {'x': [5955, 6005], 'y': [2501, 2551], 'class': 0}, {'x': [2919, 2969], 'y': [3233, 3283], 'class': -2}, {'x': [85, 135], 'y': [3980, 4030], 'class': -1}, {'x': [402, 452], 'y': [4402, 4452], 'class': 3}, {'x': [3112, 3162], 'y': [4209, 4259], 'class': 0}, {'x': [2786, 2836], 'y': [4108, 4158], 'class': -2}, {'x': [2932, 2982], 'y': [4500, 4550], 'class': -2}, {'x': [4168, 4218], 'y': [4904, 4954], 'class': 3}, {'x': [6129, 6179], 'y': [4873, 4923], 'class': 4}, {'x': [5262, 5312], 'y': [5282, 5332], 'class': -1}, {'x': [1474, 1524], 'y': [2230, 2280], 'class': -2}, {'x': [2033, 2083], 'y': [2748, 2798], 'class': -2}, {'x': [2702, 2752], 'y': [905, 955], 'class': -2}, {'x': [36, 86], 'y': [1370, 1420], 'class': -2}, {'x': [917, 967], 'y': [1589, 1639], 'class': -2}])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotations_dict[fname].values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ad3a91ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmaubreville\u001b[0m (\u001b[33mimi\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.13.5 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.12.16"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/home/aubreville/code/HierarchicalFCOS/wandb/run-20221203_140945-2ksarhpc</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href=\"https://wandb.ai/imi/HFCOS/runs/2ksarhpc\" target=\"_blank\">HFCOS-Mitosis-HE-run14</a></strong> to <a href=\"https://wandb.ai/imi/HFCOS\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<button onClick=\"this.nextSibling.style.display='block';this.style.display='none';\">Display W&B run</button><iframe src=\"https://wandb.ai/imi/HFCOS/runs/2ksarhpc?jupyter=true\" style=\"border:none;width:100%;height:420px;display:none;\"></iframe>"
      ],
      "text/plain": [
       "<wandb.sdk.wandb_run.Run at 0x7f13fd5acc70>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wandb.init(entity=\"imi\", project=\"HFCOS\", name='HFCOS-Mitosis-HE-run%d' % run)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "92e7182f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/aubreville/.local/lib/python3.8/site-packages/albumentations/augmentations/dropout/cutout.py:50: FutureWarning: Cutout has been deprecated. Please use CoarseDropout\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Labels for sampling {-1: 546, 0: 200, 1: 25, 2: 98, 3: 524, 4: 573, -2: 4139}\n",
      "Labels for sampling {-1: 128, 0: 57, 1: 3, 2: 22, 3: 150, 4: 184, -2: 1145}\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "img_dir = './'\n",
    "batch_size = 8\n",
    "crop_size = (1024,1024)\n",
    "\n",
    "num_workers = 8\n",
    "\n",
    "transform = A.Compose([\n",
    "    A.HorizontalFlip(p=0.3),\n",
    "    A.ShiftScaleRotate(p=0.5),\n",
    "    A.Cutout(p=0.2, max_w_size=100, max_h_size=100, num_holes=4),\n",
    "    A.ColorJitter(brightness=0.2, contrast=0.1, saturation=0.1, hue=0.05, p=0.3)\n",
    "], bbox_params=A.BboxParams(format='pascal_voc', label_fields = ['labels','sub_labels']))\n",
    "\n",
    "# create datamodule\n",
    "dm = MitosisTwoSubclassDataModule(img_dir,\n",
    "                               annotations_dict,\n",
    "                               labels=np.arange(-1,5), # labels for stratified sampling\n",
    "                               batch_size=batch_size,\n",
    "                               crop_size=crop_size,\n",
    "                               num_workers = num_workers,\n",
    "                               num_train_samples=2000,\n",
    "                               num_val_samples=2000,\n",
    "                               train_transforms = transform)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2bf5722c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/aubreville/.local/lib/python3.8/site-packages/pytorch_lightning/loggers/wandb.py:396: UserWarning: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.\n",
      "  rank_zero_warn(\n",
      "/home/aubreville/.local/lib/python3.8/site-packages/torchvision/models/_utils.py:135: UserWarning: Using 'weights' as positional parameter(s) is deprecated since 0.13 and may be removed in the future. Please use keyword parameter(s) instead.\n",
      "  warnings.warn(\n",
      "/home/aubreville/.local/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet18_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet18_Weights.DEFAULT` to get the most up-to-date weights.\n",
      "  warnings.warn(msg)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "/home/aubreville/.local/lib/python3.8/site-packages/pytorch_lightning/core/optimizer.py:329: RuntimeWarning: The lr scheduler dict contains the key(s) ['interval', 'frequency'], but the keys will be ignored. You need to call `lr_scheduler.step()` manually in manual optimization.\n",
      "  rank_zero_warn(\n",
      "\n",
      "  | Name   | Type                 | Params\n",
      "------------------------------------------------\n",
      "0 | model  | FCOS                 | 23.8 M\n",
      "1 | metric | MeanAveragePrecision | 0     \n",
      "------------------------------------------------\n",
      "23.8 M    Trainable params\n",
      "0         Non-trainable params\n",
      "23.8 M    Total params\n",
      "95.308    Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34dd3bbc994d4a568d8352608161bb8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Metric val/loss improved. New best score: 5.008\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Metric val/loss improved by 3.401 >= min_delta = 0.0. New best score: 1.607\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Metric val/loss improved by 0.210 >= min_delta = 0.0. New best score: 1.396\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Metric val/loss improved by 0.116 >= min_delta = 0.0. New best score: 1.281\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Metric val/loss improved by 0.035 >= min_delta = 0.0. New best score: 1.246\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Metric val/loss improved by 0.048 >= min_delta = 0.0. New best score: 1.198\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Metric val/loss improved by 0.050 >= min_delta = 0.0. New best score: 1.148\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Metric val/loss improved by 0.006 >= min_delta = 0.0. New best score: 1.141\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Monitored metric val/loss did not improve in the last 10 records. Best score: 1.141. Signaling Trainer to stop.\n"
     ]
    }
   ],
   "source": [
    "from pytorch_lightning.loggers import WandbLogger\n",
    "from pytorch_lightning.callbacks.early_stopping import EarlyStopping\n",
    "\n",
    "wandb_logger = WandbLogger(name='HFCOS_l2', project=\"HFCOS\")\n",
    "\n",
    "model = LitHFCOS(batch_size = batch_size, subclass_names=subclass_names,\n",
    "                                     num_classes = 2,optimizer='AdamW', scheduler='BaselineLRScheduler',\n",
    "                                     num_subclasses = 2,num_subclasses_l2=4, backbone='resnet18')\n",
    "model.subclass_w = [1,1]\n",
    "\n",
    "trainer = pl.Trainer(max_epochs = 50, accelerator='gpu', \n",
    "                     callbacks=[EarlyStopping(monitor=\"val/loss\", mode=\"min\", verbose=True, patience=10)],\n",
    "                     logger =wandb_logger, reload_dataloaders_every_n_epochs=1)\n",
    "\n",
    "trainer.fit(model, datamodule=dm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7c85a5df",
   "metadata": {},
   "outputs": [],
   "source": [
    "categories_l1 = {-1:'undefined',0:'atypical',1:'typical'}\n",
    "categories_l2 = {-1: 'undefined', 0:'RING SHAPE', 1:'ANAPHASE-TELOPHASE',2:'PROMETAPHASE',3:'METAPHASE'}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "67b0c9d9",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8cf8ca7b93234935b96c25283f5c1c68",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/250 [00:04<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from tqdm.auto import tqdm\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from src.utils import match_scores_targets\n",
    "model = model.eval().cuda()\n",
    "\n",
    "\n",
    "all_gt_scores = []\n",
    "all_predictions = []\n",
    "all_gt_scores_l2 = []\n",
    "all_predictions_l2 = []\n",
    "all_logits = []\n",
    "all_logits_l2 = []\n",
    "with torch.no_grad():\n",
    "    for n, (img,target) in enumerate(tqdm(dm.val_dataloader())):\n",
    "        \n",
    "        pred = model(img.cuda(),[{'boxes': y['boxes'].cuda(), 'labels': y['labels'].cuda(),\n",
    "                                  'sub_labels': y['sub_labels'].cuda(), 'sub_labels_l2': y['sub_labels_l2'].cuda()} for y in target])[0]\n",
    "        pred = [{x:y[x].cpu() for x in y } for y in pred]\n",
    "        for p,t in zip(pred,target):\n",
    "            boxes = p['boxes'].numpy()\n",
    "            scores = p['scores'].numpy()\n",
    "            scl_pred = p['sub_labels'].numpy()\n",
    "            scl_pred_l2 = p['sub_labels_l2'].numpy()\n",
    "            sclogits_pred = p['sub_logits'].numpy()\n",
    "            sclogits_pred_l2 = p['sub_logits_l2'].numpy()\n",
    "            \n",
    "            annos = t['boxes'].numpy()\n",
    "            scl_targets = t['sub_labels'].numpy()\n",
    "            scl_targets_l2 = t['sub_labels_l2'].numpy()\n",
    "            gt_scores, pred_scores,_,_ = match_scores_targets(boxes,scores,scl_pred,annos,scl_targets, radius = 25, det_th = 0.5)\n",
    "            gt_scores_l2, pred_scores_l2,_,_ = match_scores_targets(boxes,scores,scl_pred_l2,annos,scl_targets_l2, radius = 25, det_th = 0.5)\n",
    "            _, pred_logits,_,_ = match_scores_targets(boxes,scores,sclogits_pred,annos,scl_targets, radius = 25, det_th = 0.5)\n",
    "            _, pred_logits_l2,_,_ = match_scores_targets(boxes,scores,sclogits_pred_l2,annos,scl_targets_l2, radius = 25, det_th = 0.5)\n",
    "            \n",
    "            all_gt_scores.extend(gt_scores)\n",
    "            all_predictions.extend(pred_scores)\n",
    "\n",
    "            all_gt_scores_l2.extend(gt_scores_l2)\n",
    "            all_predictions_l2.extend(pred_scores_l2)\n",
    "    \n",
    "            all_logits.extend(pred_logits)\n",
    "            all_logits_l2.extend(pred_logits_l2)\n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "adb6cc32",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, '2-Level Hierarchical Classifier, AUC=0.849')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAva0lEQVR4nO3deZgU5dX38e9xxjVBVEBF9lUYcImOIiI7gkRRiY8GRdxGQHB7XKMxIhIloCAIggiIC4trJJJIgr5qYh6jIgqioCiyo8gii7ggM5z3j6oh7TgrTE11T/8+19UX3VV3d53qaerUvdRd5u6IiEj62ifuAEREJF5KBCIiaU6JQEQkzSkRiIikOSUCEZE0p0QgIpLmlAjSkJk9bmb37MX7e5vZy+UZ054ysxVm1qWIdW3NbEmU2yjFezuY2Zq9jaGYz59gZncmvB5gZl+Z2XYzqxb+2zCq7UvloERQzsxsfzN71MxWmtk3ZrbAzLqX8J49PtCUNzO7zMz+r5Dlu2N09+nu3rXioysbd/+3ux8d9XbM7GQzm21mW8zsazOba2aXR71dAHe/yt3/GMaxL/AA0NXdf+num8J/l0UZg5kNNjM3s1aFLJ9WSHk3s8YJr7uZ2Rvh/5cNZvYvMzt7D+K4KPx/962Z/cXMDiumbCcze9/MtpnZMjPrV0S5KYXE29zMXjOzrWa21Mx6ljXWZKNEUP4ygdVAe6Aq8AfgWTOrH2dQycLMMspYPjOqWMqDmbUGXgP+BTQGqgEDgGKTf0SOAA4AFu3tB5X2ezczAy4Bvg7/Let2/gd4DngSqE2wD4OAHmX8nBbAI0Cf8DO+A8YXUXZfYGZYvirwW+ABMzuuQLnTgEYFlmUCLwJ/Aw4D+gHTzKxpWeJNOu6uR8QPYCFwXjHrVwBdClm+D3Ab8DmwCXgWOCxc93fgmgLlPwB+Ez5vBrxC8B90CXBBQrnHgXuKiOUy4P+Ki7FgmVJs62FgNvAt0AU4E5gPbCNImoMTytcHHMgBVgFvhMv7Ah8D3wCLgRMS4ro5/I63As8AB4TrOgBrEj67DvACsCH8Ph8KlzciOJhvAjYC04FDSvr7hOv+DxhXzN+2YAz5f8/8/eiZsK4xQULZGsbxTLjcgFHA+vA7+xBomfi3BJqG368D24HXwvUONA6f7w+MCL/Xr4AJwIGJcQK/A9YBU0v5224HfA/0Dr+//RLWDQamFfIeD/fVwlhuKYf/Y0OBGQmvGwE/AlUKKXtEGMNBCcveBS5MeJ1J8Bs9tsB32DL8fi2h7MvAH6M8hkT9UI0gYmZ2BMF/0j05S7sWOJegdnEUsBkYF657CrgwYTtZQD3gJTP7BcGBeQZwONALGB+WKVel3NZFwL1AFYID57cEZ4+HECSFAWZ2boGPbg80B7qZ2fkEB5VLgIOBswkOOvkuAM4AGhD8x72skDgzCM7iVhIkm1rA0/mrgT8RfMfNCRLG4FLs+0FAa+D5ksom+BxoS3AmejfB2WTNcN0fCQ4qhxKcHY8Nl3clOOA2Dd93AT/df9z9U6BF+PIQd+9UyLaHhZ9xPMGBuBbB2Xe+IwnOcusRnOmWxqXAXwlOUqBsZ/JHE3zXRX5/ZnZa2ORW1OO0sGgLghMhANz9c4JE8LMzdXf/iuD/z+VmlhHW6uoR/Dbz3UBwErKwFPthBAkidcWdiSrzA9gX+H/AIyWUW0HhNYKPgc4Jr2sCOwnOVqoQHFDrhevuBaaEz38L/LvAZz0C3BU+f5ziawS5wJYCj10UUiMo5baeLGH/RwOjwuf1Cc7AGiasnwNcX8x3d3HC6/uACeHzDoRn4wQH7A1AZin+bucC80vx96kVxtqsmM/aHUMR6xcA54TPnwQmArULlOkEfAqcAuxTYN3uv2XCd5eZsD7x7PtboFHCutbA8oQ4fySsTZXy930QQQ3l3IS/+4sJ6wdTfI2gTfi81NssJpZXgasKLFsLdCiifA+CWlFu+OibsK4OsBSomhhv+HxfYBlwa/i8a/i9zdnbfYjzoRpBRMxsH2AqwY/kmoTlfw9Hcmw3s94lfEw9YGb+2Q9BYsgDjnD3b4CXCM7AIagdTE94X6vEMyeCqvuRpQz/bXc/JPFBUIUvKsaStrU68Q1m1srMXg87BrcCVwHVC3xu4nvqEJxJF2VdwvPvgF8WUqYOsNLdcwuuMLMjzOxpM1trZtuAaYXEU5jNBAmyZkkFE7Z1STiAIP+7apmwrVsJDthzzWyRmV0B4O6vAQ8R1AbXm9lEMzu4tNsM1SA4cL+XsO1/hMvzbXD3H8rwmT0JDqKzw9fTge5mlv+ZuQQHy93C9nkITmjyazWl/v6KsZ2gtpjoYIImuJ8ws2YEtcFLgP0IahO3mtmZYZHRwBB331rwve6+k+BE4UyC391NBLWhyEaGVQQlggiEHWiPErRFnhf+eABw9+4ejOT4pbtPL/JDAquB7gUOyge4+9pw/VPAhWHV9gDg9YT3/avA+37p7gPKdUdLv62CU9zOAGYBddy9KkFbtRUok/ie1RTotNvDOOsW0Qk6NNzeMe5+MHBxIfH8jLt/B7wFnFeaAMysHjCJ4MSgWphgP8rflruvc/e+7n4U0J+gia1xuG6Mu58IZBE0d9xSmm0m2EjQlt8i4e9U1d0Tk2ZZpyK+lCDprjKzdQSdvvsSNAVCcPJQv8B7GhAkiLUE/UmrKeb7s2AI8PZiHm3DoouA4xLe15CgT+TTQj62JfCpu89x913uvoTgpCq/g78zcL+ZrQv3C+AtM7sIwN0Xunt7d6/m7t2AhsDcYr+pJKdEEI2HCdqae7j796V8z75mdkDCI5PgAHlveADBzGqY2TkJ75lNcEY+hKBjcVe4/G9AUzPrY2b7ho+TzKx5uezdT+3JtqoAX7v7D2Z2Mv89cBRlMnCzmZ1ogcb530kZzAW+BIaZ2S/C77hNQjzbga1mVouyHWRvBS4zs1vMrBqAmR1nZk8XUvYXBAfbDWG5y0loWzaz882sdvhyc1h2V/h9tgrPpr8FfiCoiZRa+NuYBIwys8PD7dUys27Fvc+CoZMdCllei+CAeRZBn8PxBAfi4fx39NA/gGYJv43DCJLun90914O2lhuBO83scjM72Mz2CfsFJoZx/zvhxKmwx7/DbU0HeoSJ4xcE/ydeCGvOBc0HmlgwhNTMrFG4H/n9AU3DfcnfLwiakmaG+35s+Ps5yMxuJqjRPF7c95jslAjKWXiA6k/wA1pXhmag2QRnbPmPwcCDBGfOL5vZN8DbwO6x2u6+g2AUTBeCs+z85d8QtF32Ar4gqMIOJzhDKld7uK2BwJBwnwbx347GorbxHEEfyAyCqv5fCDo1yxJnHsF/5sYEZ6prCPo3IOi0PYFgtM5LBN9paT/3PwRt+J2AZWb2NUE7/+xCyi4GRhLUIr4CjgHeTChyEvCOmW0n+Ltf78E1AAcTHMQ3E3R2bwLuL22MCX5H0Pb9dtgE9v8IOmwLZWZ1CL7vDwtZ3QdY4O4vhzWZde6+DhgDHGtmLd19PcFZdn+CEU8fEfQ37a4tuvvzBH+HKwh+P18RjIJ6sSw75u6LCJoYp4fbqkLwO8vfl7+b2e/Dsp+H2xtD0MfxL+DPBCccuPv6AvsEsDHhpK4PwUnFeoJkeHr4fzFlWZCURUR+yswuJmhKuj3uWCRaSgQiImlOTUMiImlOiUBEJM0pEYiIpLmkntCrMNWrV/f69evHHYaISEp57733Nrp7jcLWpVwiqF+/PvPmzYs7DBGRlGJmK4tap6YhEZE0p0QgIpLmlAhERNKcEoGISJpTIhARSXORJQILbvq83sw+KmK9mdkYC27+vNDMTogqFhERKVqUNYLHCW4fWJTuQJPw0Y9g6mYREalgkV1H4O5vmFn9YoqcQ3ALQyeYFvcQM6vp7l9GFZOIpK8Z76zixQVrSy6YhHJ3fM+Ob7Zw0rFHc1ePFiW/oYzivKCsFj+9HeGacNnPEoGZ9SO8mXbdunUrJDgRiV95HrzfWf41AK0alOlWFrH76pN5zJs2jH0P/CUnjn0+km2kxJXF7j6R4GYfZGdna95skSRV3mfd5XnwbtXgMM45vhYXtUqNk8ktW7Zwyy238OzkyTRu3JjJkyfSvv0xkWwrzkSwluCG4vlqh8tEJIUkHvzL+6w71Q7e5SUvL49TTz2VJUuWcOuttzJ48GAOPPDAyLYXZyKYBVwT3tu1FbBV/QMiyaO0Z/eJB/90PXCXl02bNnHYYYeRkZHBvffeS506dcjOzo58u5ElAjN7CugAVDezNcBdwL4A7j6B4J6uvya4h+p3wOVRxSIipbMnZ/c6+O89d2f69Olcf/31DBs2jL59+9KzZ88K236Uo4YuLGG9A1dHtX2RdLcn7fU6u694q1ev5qqrrmL27NmccsoptGnTpsJjSInOYhEpXHEH+z1pr9fBv2I99dRT9O/fn7y8PEaPHs0111xDRkZGhcehRCCSIgo76Bd3sNdBPfkdeuihtGrViokTJ9KgQYPY4rCghSZ1ZGdnu25MI+lmxjur+P3MD4GfH/R1sE8dubm5jBo1ih9//JE77rgDCPoHzCzybZvZe+5eaM+zagQiSS4xCQzteYwO+inqgw8+ICcnh/fee48LLrhgdwKoiCRQEiUCkSRQmrZ+JYHUtGPHDu655x6GDRvGYYcdxnPPPcd5552XFAkgnxKBSAUoaQSP2vorr88++4zhw4dz0UUX8cADD1CtWrW4Q/oZJQKRiBXXvp9PB/vKZfv27bz44ov07t2bli1b8sknn9CwYcO4wyqSEoFIBAq7MEtNO+nhlVdeoV+/fqxcuZITTjiB5s2bJ3USACUCkXJRsOlHF2aln82bN3PzzTczZcoUmjZtyr/+9S+aN28ed1ilokQgspcKa/rRwT+95OXl0aZNGz799FNuv/12Bg0axAEHHBB3WKWmRCBSRkWd/avpJ/1s3Lhx9yRxQ4cOpW7dupxwQurddVeJQKSAso7w0dl/+nF3pk6dyv/+7/8ybNgw+vXrx7nnnht3WHtMiUDSXnHt+4XRgT+9rVy5kv79+zNnzhxOPfVU2rVrF3dIe02JQNJKaebr0YFeijJt2jQGDBiAuzN27FgGDhzIPvvsE3dYe02JQCqt0k7SpgO/lFaNGjVo06YNjzzyCPXq1Ys7nHKjSeckpe3JNMw66Etp7dy5k5EjR7Jz507uvPNOoOImiStvmnROKgVNwywVaf78+eTk5DB//nx69eqVVJPElTclAklqJd06UQd7KW8//PADQ4YM4b777qN69er8+c9/5je/+U3cYUVKiUCSTlEHfx30pSIsXbqUESNGcMkllzBy5EgOPfTQuEOKnBKBxE7TM0jctm/fzsyZM+nTpw8tW7ZkyZIlsd4xrKIpEUiFK2ncvg7+UpHmzJlDv379WL16NdnZ2TRv3jytkgAoEUgF0IFfktGmTZu48cYbefLJJ2nWrBn//ve/U2aSuPKmRCCRe3HBWhZ/uY2smgcDOvBL/PIniVu6dCl33HEHf/jDH1JqkrjypkQgkUisBeQngWf6t445Kkl3GzZsoFq1amRkZDB8+HDq1avH8ccfH3dYsUv9a6Ml6eRPy5zfBJRV82DOOb5WzFFJOnN3HnvsMZo2bcqkSZMAOOecc5QEQqoRSLnQHbkkWa1YsYJ+/frxyiuv0LZtWzp27Bh3SElHiUD2WsEbs6gPQJLF1KlTGTBgAGbG+PHj6d+/f6WYJK68KRHIHsuvBagGIMnqiCOOoF27dkyYMIG6dfXbLIoSgeyx/NFAqgFIsti5cyf33XcfeXl5DBo0iK5du9K1a9e4w0p6SgRSKoVN+KbRQJJM3n//fa644go++OADLrroopSdJTQOaiyTEhUcBZRPo4EkGXz//ffcdtttnHzyyXz11VfMnDmT6dOnKwmUQaQ1AjM7A3gQyAAmu/uwAuvrAk8Ah4RlbnP32VHGJGWT2BGsPgBJRsuWLeOBBx7gsssu4/7770+LSeLKW2Q1AjPLAMYB3YEs4EIzyypQ7A/As+7+K6AXMD6qeGTP5DcHKQlIMtm2bRuPP/44AC1atOCzzz5j8uTJSgJ7KMqmoZOBpe6+zN1/BJ4GzilQxoGDw+dVgS8ijEfKaMY7q3hn+de0anCYkoAkjdmzZ9OyZUtycnL4+OOPASrVbSPjEGXTUC1gdcLrNUCrAmUGAy+b2bXAL4AuhX2QmfUD+gEaAlYBCg4LVT+AJIONGzdyww03MG3aNLKysnjzzTfTdpK48hZ3Z/GFwOPuXhv4NTDVzH4Wk7tPdPdsd8+uUaNGhQeZbhKHhapJSJJB/iRxTz/9NIMGDeL999/nlFNOiTusSiPKGsFaoE7C69rhskQ5wBkA7v6WmR0AVAfWRxiXFCG/JqBhoZIsvvrqK2rUqEFGRgYjRoygXr16HHvssXGHVelEWSN4F2hiZg3MbD+CzuBZBcqsAjoDmFlz4ABgQ4QxSRESh4hqWKjEzd159NFHOfroo5k4cSIAPXr0UBKISGQ1AnfPNbNrgDkEQ0OnuPsiMxsCzHP3WcBNwCQzu4Gg4/gyd/eoYpLCaYioJJNly5bRt29fXnvtNdq3b0+XLoV2HUo5ivQ6gvCagNkFlg1KeL4YaBNlDFI0zRUkyeaJJ55g4MCBZGRkMGHCBPr27atJ4iqApphIQwUTgOYKkmRx1FFH0alTJx5++GFq164ddzhpQ4kgDWmyOEkWP/74I8OGDWPXrl0MHjyY008/ndNPPz3usNKOEkEa0aggSSbvvvsuV1xxBR999BF9+vTRJHExUiJIA0U1BYnE4bvvvmPQoEGMGjWKmjVrMmvWLHr06BF3WGlNiaCSK3j3MDUFSdyWL1/O2LFj6du3L8OHD6dq1apxh5T2lAgqOU0aJ8lg69atvPDCC1x++eW0aNGCpUuXUqdOnZLfKBVC47IqqRnvrOK3j7y1u1NYSUDi8tJLL9GiRQuuvPJKPvnkEwAlgSSjRFBJJXYKqz9A4rBhwwZ69+7NWWedxaGHHspbb71Fs2bN4g5LCqGmoUoocfpojQySOOTl5XHaaaexfPly7r77bm677Tb222+/uMOSIigRVCKaPlritm7dOg4//HAyMjIYOXIk9evXp2XLlnGHJSVQIkhxiTeV15XCEpddu3YxadIkbrnlFoYPH86AAQM466yz4g5LSkmJIMUl9gUoAUgcli5dSt++ffnnP/9Jp06d6NatW9whSRkpEaQw9QVI3B577DEGDhzIfvvtx6RJk8jJydHVwSlIiSBFJV4opr4AiUvdunXp1q0b48aNo1Yt/Q5TlRJBCtL9AyQuO3bs4E9/+hO7du1iyJAhdO7cmc6dO8cdluwlXUeQYpQEJC7vvPMOJ554InfffTerVq1C95CqPJQIUoiSgMTh22+/5cYbb6R169Zs3bqVv/3tbzz++OPqC6hElAhSiOYNkjisXLmS8ePHc9VVV7Fo0SLOPPPMuEOScqY+ghSROEJISUCitmXLFp5//nmuvPJKsrKyWLp0qe4YVompRpACNEJIKtKLL75IVlYWV1111e5J4pQEKjfVCJJUYVcMq0lIorR+/Xquu+46nnnmGY499lhmzZqlSeLShBJBktIVw1KR8vLyaNOmDatWreKee+7h1ltvZd999407LKkgSgRJTPcVlqh98cUXHHnkkWRkZPDggw9Sv359srKy4g5LKpj6CETS0K5du3j44Ydp1qwZEyZMAODXv/61kkCaUiJIQvkjhESi8Omnn9KxY0cGDhxIq1at6N69e9whScyUCJJQfiexRghJeXv00Uc57rjjWLhwIVOmTOHll1+mQYMGcYclMVMfQZLR9QISpfr169O9e3fGjRtHzZo14w5HkoQSQZJRbUDK044dO/jjH/8IwD333KNJ4qRQSgRJIPGagcVfblNtQMrFf/7zH3Jycvjkk0+44oorcHfNDySFUh9BzPKvGs7vHM6qebBqA7JXtm/fzvXXX89pp53Gd999xz/+8Q8effRRJQEpUqQ1AjM7A3gQyAAmu/uwQspcAAwGHPjA3S+KMqZko4nkpLytWrWKRx55hKuvvpqhQ4dSpUqVuEOSJBdZIjCzDGAccDqwBnjXzGa5++KEMk2A24E27r7ZzA6PKp5kk98cpKYgKQ+bN2/mueeeo1+/fmRlZbFs2TKOOuqouMOSFBFl09DJwFJ3X+buPwJPA+cUKNMXGOfumwHcfX2E8SSVxCkk1BQke2PmzJlkZWUxcOBAlixZAqAkIGUSZSKoBaxOeL0mXJaoKdDUzN40s7fDpqSfMbN+ZjbPzOZt2LAhonArXv4UEqoNyJ5Yt24d559/Pr/5zW848sgjmTt3LkcffXTcYUkKinvUUCbQBOgA1AbeMLNj3H1LYiF3nwhMBMjOztb98STt5eXl0bZtW1avXs3QoUO5+eabNUmc7LEoE8FaoE7C69rhskRrgHfcfSew3Mw+JUgM70YYl0jKWrNmDUcddRQZGRmMGTOGBg0aaKpo2WtRNg29CzQxswZmth/QC5hVoMxfCGoDmFl1gqaiZRHGlBQ0l5CU1a5duxg7dizNmjXj4YcfBqB79+5KAlIuIksE7p4LXAPMAT4GnnX3RWY2xMzODovNATaZ2WLgdeAWd98UVUzJQHcbk7L65JNPaNeuHddddx2nnXYaZ511VtwhSSVj7qnV5J6dne3z5s2LO4w9kpgEdN2AlMbkyZO55pprOOiggxg9ejR9+vTRhWGyR8zsPXfPLmxd3J3FaUNJQPZEo0aN6NGjBw899BBHHHFE3OFIJaVEUEF0BbGUxg8//MCQIUMAGDp0KB07dqRjx44xRyWVneYaqkC6gliK8+abb3L88cfzpz/9iQ0bNpBqzbaSupQIRGL2zTffcO2119K2bVt27NjBnDlzmDRpkvoCpMIoEYjEbM2aNUyePJlrr72WDz/8kK5du8YdkqQZ9RFEqOB9BrJqHhxzRJIsNm3axLPPPsuAAQNo3rw5y5Yt0x3DJDaqEUQof2I50H0GJODuPP/882RlZXHdddftniROSUDitMc1AjOr6+6ryjOYyih/YjmRL7/8kquvvpqZM2dy4okn8vLLL2uSOEkKJSYCM2tNMGvoG+6+3syOBW4D2vLTuYREpAj5k8StXbuW++67jxtuuIHMTLXMSnIo9pdoZvcDZwELgN+Z2RzgSuBPwBWRRyeS4lavXk2tWrXIyMhg3LhxNGjQgKZNm8YdlshPlNRHcCbwK3e/EOgK/C9wirs/6O4/RB1cKtPEcuktLy+PMWPG/GSSuG7duikJSFIqqW76Q/4BP7yV5GfuviL6sFJf/mghdRCnn48//picnBzeeustunfvTo8ePeIOSaRYJSWChmaWP3W0AQ0SXuPuZxf+tvSWXxvQlcTpZ+LEiVx77bVUqVKFqVOn0rt3b10YJkmvpERQ8B7DI6IKpDJRbSB9NWnShJ49ezJmzBgOP/zwuMMRKZViE4G7/8vMjgcaA4vc/eMKiaoSUG0gPXz//fcMHjwYM2PYsGGaJE5SUrGdxWY2CHgWOA94ycz6VkhUIingjTfe4LjjjuO+++5j69atmiROUlZJo4Z+Cxwfjho6CegXfUgiyW3btm0MHDiQ9u3bk5eXx6uvvsrDDz+svgBJWSUlgh3u/h1AeAtJTUkhae+LL77g8ccf58Ybb2ThwoV06tQp7pBE9kpZRw010qih4iWOGJLKY+PGjTz77LMMHDiQZs2asXz5ct0xTCoNjRoqZxoxVLm4O88++yzXXnstW7ZsoUuXLjRt2lRJQCqVkhLBck0sV3q6fqBy+eKLLxgwYACzZs0iOzubV199VVcGS6VUUpv/X/KfmNmfow0l9ak2UHnk5eXRrl07Xn75ZUaMGMFbb73FMcccE3dYIpEoqUaQOAyiYZSBpDrVBiqHlStXUrt2bTIyMhg/fjwNGzakcePGcYclEqmSagRexHNJMOOdVfx+5oeAagOpKi8vjwceeIDmzZvvniSua9euSgKSFkqqERxnZtsIagYHhs8JX7u7696L/LdJaGjPY1QbSEEfffQROTk5zJ07l7POOotzzz037pBEKlRJU0xkVFQgqU5NQqlpwoQJXHfddVStWpUZM2bQq1cvXRgmaUe3SNpDujF9anN3zIzmzZtz/vnnM3r0aGrUqBF3WCKxUCLYA4l9Aq0aHKYb06eQ7777jkGDBpGRkcHw4cNp37497du3jzsskVgpEZRBfi0g/85j6hNILf/85z+58sor+fzzzxk4cODuWoFIutPcQWXw4oK1LP5yG60aHKYkkEK2bt1K//79d08P/dprrzFu3DglAZGQagRllFXzYJ7p3zruMKQMvvzyS6ZNm8bNN9/M3XffzUEHHRR3SCJJJdIagZmdYWZLzGypmd1WTLnzzMzNLDvKeCR9bNiwgbFjxwLQrFkzVqxYwf33368kIFKIyBKBmWUA44DuQBZwoZllFVKuCnA98E5UsUj6cHdmzJhB8+bNuemmm/j0008BNCJIpBhR1ghOBpa6+zJ3/xF4mp/PZgrwR2A48EOEsUgaWL16NT169KB37940btyY+fPna5I4kVKIMhHUAlYnvF4TLtvNzE4A6rj7S8V9kJn1M7N5ZjZvw4YN5R+ppLzc3Fw6dOjA66+/zqhRo3jzzTdp0aJF3GGJpITYOovNbB/gAeCyksq6+0RgIkB2drbmPJLdVqxYQZ06dcjMzOSRRx6hYcOGNGyo+RFFyiLKGsFaoE7C69rhsnxVgJbAP81sBXAKMEsdxlIaubm5jBgxgubNmzN+/HgAunTpoiQgsgeiTATvAk3MrIGZ7Qf0AhJvc7nV3au7e313rw+8DZzt7vMijGmP5U8zLfFbuHAhrVu35pZbbqFbt26cd955cYckktIiSwTungtcA8wBPgaedfdFZjbEzFLuXse66UxyGD9+PCeeeCIrV67kmWeeYebMmRx11FFxhyWS0iLtI3D32cDsAssGFVG2Q5Sx7A3ddCZ++dNBtGzZkl69ejFq1CiqV68ed1gilYKuLC4F1Qbi8+233/KHP/yBzMxM7r//ftq1a0e7du3iDkukUtFcQ6Wk2kDFe/XVVznmmGMYPXo0O3bswF0DxkSioERQAnUSV7wtW7Zw5ZVX0qVLFzIzM3njjTcYM2aMJokTiYgSQQnULFTxvvrqK55++ml+97vf8cEHH9C2bdu4QxKp1NRHUAx1Elec/IP/9ddfz9FHH82KFSvUGSxSQVQjKIZqA9Fzd6ZNm0ZWVha33norn332GYCSgEgFUiIogWoD0Vm1ahVnnnkmffr04eijj2bBggU0adIk7rBE0o6ahiQW+ZPErV+/njFjxjBw4EAyMjLiDkskLSkRFCGxf0DKz7Jly6hXrx6ZmZlMmjSJRo0aUb9+/bjDEklrahoqgvoHyldubi7Dhw8nKyuLcePGAdC5c2clAZEkoBpBMdQ/UD4WLFhATk4O77//Pj179uT888+POyQRSaAagUTqoYce4qSTTmLt2rU8//zzvPDCC9SsWTPusEQkgRKBRCJ/Oohjjz2W3r17s3jxYk0XLZKk1DQk5Wr79u3ccccd7LvvvowYMUKTxImkANUIpNy8/PLLtGzZkrFjx7Jz505NEieSIpQICqGJ5spm8+bNXH755XTr1o0DDjiAN954gwcffFCTxImkCCWCAma8s4rfz/wQ0NDR0lq/fj3PP/88t99+OwsWLOC0006LOyQRKQP1EYRmvLOKFxes3V0TGNrzGA0dLca6det46qmnuOGGG3ZPEletWrW4wxKRPaAaQejFBWtZ/OU2WjU4TEmgGO7OE088QVZWFrfffvvuSeKUBERSlxJBgqyaB/NM/9ZKAkVYsWIFZ5xxBpdddhlZWVmaJE6kklDTkJRKbm4uHTt2ZOPGjYwbN46rrrqKffbReYRIZaBEIMVaunQpDRo0IDMzkylTptCwYUPq1asXd1giUo50SoeGixZm586dDB06lBYtWuyeJK5jx45KAiKVUFrXCAqOFNJw0cD7779PTk4OCxYs4Pzzz+e3v/1t3CGJSITSOhEkjhQ65/ha6iQGxowZw4033kiNGjV44YUX6NmzZ9whiUjE0jYRJN545pn+reMOJ3bujpnxq1/9iksuuYSRI0dy6KGHxh2WiFSAtE0EuvFM4JtvvuH2229n//33Z+TIkbRt25a2bdvGHZaIVKC07ixO9xvP/OMf/6Bly5aMHz8ed9ckcSJpKq0TQbratGkTl156Kd27d+cXv/gFb775Jg888IAmiRNJU2mZCNJ9uOimTZuYOXMmd955J/Pnz6d1a/WRiKSzSBOBmZ1hZkvMbKmZ3VbI+hvNbLGZLTSzV82sQgapp2P/wJdffsmIESNwd5o2bcrKlSsZMmQI+++/f9yhiUjMIksEZpYBjAO6A1nAhWaWVaDYfCDb3Y8FngfuiyqegtKlf8DdmTJlCs2bN+fOO+9k6dKlABoRJCK7RVkjOBlY6u7L3P1H4GngnMQC7v66u38XvnwbqB1hPEB6NQstX76crl27kpOTw3HHHccHH3ygSeJE5GeiHD5aC1id8HoN0KqY8jnA3wtbYWb9gH4Adevu3Vl8ujQL5ebm0qlTJzZt2sTDDz9Mv379NEmciBQqKa4jMLOLgWygfWHr3X0iMBEgOzt7j8Y45k8nkX8lcWVtFvrss89o2LAhmZmZPPbYYzRq1Ig6derEHZaIJLEoTxHXAolHoNrhsp8wsy7AHcDZ7r4jqmDyk0BWzYMrZW1g586d3HPPPbRs2ZKHHnoIgA4dOigJiEiJoqwRvAs0MbMGBAmgF3BRYgEz+xXwCHCGu6+PMBbgvzeeqWzmzZtHTk4OCxcupFevXlx44YVxhyQiKSSyGoG75wLXAHOAj4Fn3X2RmQ0xs7PDYvcDvwSeM7MFZjYrqngqqwcffJBWrVqxceNGXnzxRZ566ikOP/zwuMMSkRQSaR+Bu88GZhdYNijheZcot1+Z5U8Sl52dTU5ODvfddx+HHHJI3GGJSApKis5iKb1t27bxu9/9jgMOOIBRo0bRpk0b2rRpE3dYIpLCNJ4whcyePZsWLVowceJEMjMzNUmciJQLJYIUsHHjRi6++GLOPPNMqlatyn/+8x/uv/9+TRInIuVCiSAFbN68mb/+9a/cddddvP/++7RqVdx1eSIiZaM+giS1du1apk+fzi233EKTJk1YuXKlOoNFJBKqESQZd2fSpElkZWUxePBgPv/8cwAlARGJjBJBEvn888/p3Lkz/fr144QTTmDhwoU0btw47rBEpJJT01CSyM3NpXPnznz99dc88sgjXHnllZokTkQqhBJBzJYsWUKjRo3IzMzkiSeeoFGjRtSuHfls3CIiu+mUMyY//vgjd999N8cccwzjxo0DoH379koCIlLhVCOIwdy5c8nJyeGjjz7ioosuonfv3nGHJCJpTDWCCjZ69Ghat269+9qA6dOnU7169bjDEpE0pkRQQfKngzj55JPp27cvixYt4qyzzoo5KhERNQ1FbuvWrdx6660ceOCBjB49mlNPPZVTTz017rBERHZTjSBCf/3rX8nKymLy5Mnsv//+miRORJKSEkEENmzYwEUXXcTZZ59NtWrVePvttxk+fLgmiRORpKREEIGtW7cye/Zs7r77bubNm8dJJ50Ud0giIkVSH0E5Wb16NdOmTeO2226jcePGrFy5kqpVq8YdlohIiVQj2Eu7du1iwoQJtGjRgnvuuWf3JHFKAiKSKpQI9sJnn31Gp06dGDBgACeffDIffvihJokTkZSjpqE9lJuby+mnn86WLVt49NFHufzyy9UZLCIpSYmgjD7++GOaNGlCZmYmU6dOpVGjRhx11FFxhyUissfUNFRKO3bs4K677uLYY4/loYceAqBt27ZKAiKS8lQjKIW3336bnJwcFi9eTJ8+fejTp0/cIYmIlBvVCEowcuRITj31VL755htmz57Nk08+SbVq1eIOS0Sk3CgRFGHXrl0AtG7dmquuuoqPPvqI7t27xxyViEj5U9NQAVu2bOGmm27ioIMOYuzYsZokTkQqPdUIEvzlL38hKyuLJ554gipVqmiSOBFJC0oEwPr167ngggvo2bMnRxxxBHPnzmXo0KG6LkBE0oISAbBt2zZeeeUV7r33XubOncsJJ5wQd0giIhUmbfsIVq1axdSpU/n9739P48aNWbVqFVWqVIk7LBGRChdpjcDMzjCzJWa21MxuK2T9/mb2TLj+HTOrH2U8EIwGGj9+PC1atGDo0KG7J4lTEhCRdBVZIjCzDGAc0B3IAi40s6wCxXKAze7eGBgFDI8qHoBt61bSoUMHrr76alq3bs2iRYs0SZyIpL0om4ZOBpa6+zIAM3saOAdYnFDmHGBw+Px54CEzM49guM6uvFzeGHMD++36gccee4xLL71UncEiIkSbCGoBqxNerwFaFVXG3XPNbCtQDdiYWMjM+gH9AOrWrbtHwbSscxhVbr2Pey/pQs2aNffoM0REKqOU6Cx294nARIDs7Ow9qi3c1aMF9GhRrnGJiFQGUXYWrwXqJLyuHS4rtIyZZQJVgU0RxiQiIgVEmQjeBZqYWQMz2w/oBcwqUGYWcGn4/H+A16LoHxARkaJF1jQUtvlfA8wBMoAp7r7IzIYA89x9FvAoMNXMlgJfEyQLERGpQJH2Ebj7bGB2gWWDEp7/AJwfZQwiIlI8TTEhIpLmlAhERNKcEoGISJpTIhARSXOWaqM1zWwDsHIP316dAlctpwHtc3rQPqeHvdnneu5eo7AVKZcI9oaZzXP37LjjqEja5/SgfU4PUe2zmoZERNKcEoGISJpLt0QwMe4AYqB9Tg/a5/QQyT6nVR+BiIj8XLrVCEREpAAlAhGRNFcpE4GZnWFmS8xsqZndVsj6/c3smXD9O2ZWP4Ywy1Up9vlGM1tsZgvN7FUzqxdHnOWppH1OKHeembmZpfxQw9Lss5ldEP6tF5nZjIqOsbyV4rdd18xeN7P54e/713HEWV7MbIqZrTezj4pYb2Y2Jvw+FprZCXu9UXevVA+CKa8/BxoC+wEfAFkFygwEJoTPewHPxB13BexzR+Cg8PmAdNjnsFwV4A3gbSA77rgr4O/cBJgPHBq+PjzuuCtgnycCA8LnWcCKuOPey31uB5wAfFTE+l8DfwcMOAV4Z2+3WRlrBCcDS919mbv/CDwNnFOgzDnAE+Hz54HOltp3si9xn939dXf/Lnz5NsEd41JZaf7OAH8EhgM/VGRwESnNPvcFxrn7ZgB3X1/BMZa30uyzAweHz6sCX1RgfOXO3d8guD9LUc4BnvTA28AhZrZXN2KvjImgFrA64fWacFmhZdw9F9gKVKuQ6KJRmn1OlENwRpHKStznsMpcx91fqsjAIlSav3NToKmZvWlmb5vZGRUWXTRKs8+DgYvNbA3B/U+urZjQYlPW/+8lSomb10v5MbOLgWygfdyxRMnM9gEeAC6LOZSKlknQPNSBoNb3hpkd4+5b4gwqYhcCj7v7SDNrTXDXw5buvivuwFJFZawRrAXqJLyuHS4rtIyZZRJUJzdVSHTRKM0+Y2ZdgDuAs919RwXFFpWS9rkK0BL4p5mtIGhLnZXiHcal+TuvAWa5+053Xw58SpAYUlVp9jkHeBbA3d8CDiCYnK2yKtX/97KojIngXaCJmTUws/0IOoNnFSgzC7g0fP4/wGse9sKkqBL32cx+BTxCkARSvd0YSthnd9/q7tXdvb671yfoFznb3efFE265KM1v+y8EtQHMrDpBU9GyCoyxvJVmn1cBnQHMrDlBIthQoVFWrFnAJeHooVOAre7+5d58YKVrGnL3XDO7BphDMOJgirsvMrMhwDx3nwU8SlB9XErQKdMrvoj3Xin3+X7gl8BzYb/4Knc/O7ag91Ip97lSKeU+zwG6mtliIA+4xd1TtrZbyn2+CZhkZjcQdBxflsondmb2FEEyrx72e9wF7Avg7hMI+kF+DSwFvgMu3+ttpvD3JSIi5aAyNg2JiEgZKBGIiKQ5JQIRkTSnRCAikuaUCERE0lylGz4qEgUzqwa8Gr48kmBoZv5Y9eMIJkPLBD4GLnX378wsD/gwXL4c6FPJr/CVFKXhoyJlZGaDge3uPiJ8vd3dfxk+nw685+4PFFj+BPCpu98bV9wiRVHTkEj5+jfQuJDlb7GXE4OJREWJQKSchPNWdSdoDkpcnkEwBUKlu9pZKgclApG9d6CZLQDmEcx782iB5euAI4BXYolOpATqLBbZe9+7+/FFLTezgwjmyrkaGFOhkYmUgmoEIhEL7wx3HXBT2HwkklSUCEQqgLvPBxYS3ERFJKlo+KiISJpTjUBEJM0pEYiIpDklAhGRNKdEICKS5pQIRETSnBKBiEiaUyIQEUlz/x+7cFTlwkwg3QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "sel = np.stack(all_gt_scores)>=0\n",
    "from sklearn.metrics import roc_curve\n",
    "fpr,tpr,thrs = roc_curve(y_score=np.stack(all_logits)[sel,1], y_true=np.stack(all_gt_scores)[sel])\n",
    "plt.plot(fpr,tpr)\n",
    "plt.xlabel('TPR')\n",
    "plt.plot([0,1],[0,1],'k--')\n",
    "plt.ylabel('FPR')\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "auc = roc_auc_score(y_score=np.stack(all_logits)[sel,1], y_true=np.stack(all_gt_scores)[sel])\n",
    "plt.title('2-Level Hierarchical Classifier, AUC=%.3f' % auc)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "84cb72a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix,ConfusionMatrixDisplay\n",
    "cm1 = confusion_matrix(y_true=all_gt_scores, y_pred=all_predictions)\n",
    "cm2 = confusion_matrix(y_true=all_gt_scores_l2, y_pred=all_predictions_l2)\n",
    "#\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6461f3d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_values(['N/A', 'AMF', 'RING SHAPE', 'ANAPHASE-TELOPHASE', 'PROMETAPHASE', 'METAPHASE'])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "value_to_class.values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e19619ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7f10f40399d0>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAEvCAYAAACDj+ACAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAuGElEQVR4nO3deZyVZf3/8dd7BhgEZYbNDVBQUUNzRVwzcsklS/NnmfrNtcwtl7LU6ptm5ddWK3OJ1NTcLVJyQ0VxywXcUHEjVHaVXQSEmfn8/rjvgQMOM4dhZs59H97Px+N+zLmve/ucI37Oda77uq9LEYGZmWVbRakDMDOz5jlZm5nlgJO1mVkOOFmbmeWAk7WZWQ44WZuZ5UCHUgeQZ51UFZ3pWuowMkudq0odQubVV1WWOoRMW7xwDkuXfKw1OccBX+gas2bXFbXv8+M+GRkRB67J9dqKk/Ua6ExXdtW+pQ4jsyo337LUIWTewv7VpQ4h01588k9rfI6Zs2v5zwN9itq388bv9FrjC7YRJ2szK2sB1JP/h/+crM2s7NVTX+oQ1piTtZmVtSCoK4NhNZyszazsuRnEzCzjAqhzsjYzy7YAlobbrM3MMi//qdrJ2szKXBBuBjEzy7yAuvznaidrMytvyUMx+edkbWZlLRBLY42GF8kEJ2szK3t1OFmbmWVa0s/aydrMLPPq3QxiZpZtrlmbmeVAIOrKYFIsJ2szK2vJ4+ZO1mZmGSfqnKzNzLIteSjGydrMLPN8g9HMLOMi3AxiZpYL9WVQs87/142ZWRMCsSQ6FLU0R9J1kj6Q9GpB2W8kvSFpnKR/Saop2HaBpAmS3pR0QEH5gWnZBEnnF/M+nKzNrKw13GAsZinC9cCBK5U9BGwbEdsBbwEXAEgaBHwD2CY95kpJlZIqgSuAg4BBwFHpvk1ysjazslcXKmppTkQ8DsxeqezBiKhNV58B+qavDwVui4hPIuIdYAIwJF0mRMTEiFgC3Jbu2yS3WZtZWWvnJxhPBG5PX/chSd4NpqRlAJNXKt+1uRM7WZtZ2asvvjdIL0ljC9aHRcSwYg6U9GOgFrh5NcMripO1mZW1esSSqCx295kRMXh1ryHpeOAQYN+IaJhEbCrQr2C3vmkZTZSvktuszazsteINxk+RdCDwQ+ArEbGwYNMI4BuSqiQNAAYCzwFjgIGSBkjqRHITckRz13HNukwMHjqfU34+jcqK4P5be3DHnzcodUjt7uxzxzBk1+nMnVvFad9e1kuKLx/2Nod85b/U14sxz27EdX/djg4d6vnu2c8zcKvZ1NeLv1y5A6+8vH4Jo297nTrU8sfz7qVjhzoqK+p57PkBXD9iZ776hdc4Yv/X6LP+fA49+3+Yt6AzAEceMI79d50AQGVlsMlGcznsnGP46OPOpXwbqy2CVnsoRtKtwFCS5pIpwIUkvT+qgIckATwTEadExGuS7gDGkzSPnB4Rdel5zgBGApXAdRHxWnPXzmyyljQaODcixjaxz+eAq4GlwJeAP0bEEa1w7aHptQ9Z03O1h4qK4PRLpnLBNzZj5vSOXH7f2zwzsppJb+frf6o19fDI/vz7ri34/nnPLSvbbvsP2G2PaZz+nf2pXVpJdc1iAA48eCIAp337AKprFnPxJU9w9un7EWUwSP2qLKmt5Hu/PZhFn3SksrKey8/7N8+92o9XJmzA0+M24Q8/uHeF/W8fuR23j9wOgN23f4+v7fdq7hJ1Qq32UExEHNVI8bVN7P9L4JeNlN8H3Lc61857M8gxwP9FxA4RMbU1EnUebbXjQqa924kZk6qoXVrB6Ltr2P2AeaUOq929+kpvPvqo0wplX/rKf7nztq2pXZq0Wc6bmySbTTadz8svrb+s7OMFnRi45Zz2DbjdiUWfdASgQ2U9HSrriYAJk3sxY9Z6TR6575CJjHpu8/YIstUFSc26mCXL2iw6Sf1XesrnXEkXSRot6VeSnpP0Vlo7RtI6km6T9LqkfwHrFBz7RUlPS3pB0p2S1pX0LeDrwM8l3Vx4PUnHSxou6QFJb0v6dVPnSssPTJ9CegE4vK0+l7bQc8OlfDhteZKaOb0jvTZaWsKIsmPjPh+xzbYzuezyUfzqd48ycKuki+zEiTXsuvs0Kirq2WDDj9liyzn0Xn9hM2fLvwrVc81Ph3PX729i7Pg+vP5O800/VZ1qGbLtFB5/oX/bB9hG6qgoasmyUjWDdIiIIZIOJmnz2Q84FVgYEZ+RtB3wAoCkXsBPgP0i4mNJ5wHfi4iLJe0F3BMR/5DUf6Vr7ADsCHwCvCnpcmBRY+dKk/lfgX1IOq7fjpWFyspgvW5LOOe7+7DlVnO44CdPc+I3D+bB+/vTb5P5/PHKh/ngg668/lpP6uvKtwmkQX1U8K2LD2fddT7h56c/zICNZ/POtB5NHrPH9u/x6oT1c9oEkvSzXlp8b5DMKlWyHp7+fR7on77eG/gTQESMkzQuLd+N5JHMp9LG+07A00VcY1REzAOQNB7YFKhZxbm2Bt6JiLfT/W8CTm7spJJObtjWmS7FvNc2N2tGR3pvvGTZeq+NljJzescSRpQdM2euw3+e6AOIt97sQYToVr2E+fOq+OtVOyzb77d/fIQpU5puCignCxZV8eIbGzFk2ynNJut9dpnIqGfz2QQC6ePmGW/iKEZbvoPalc5f+LX8Sfq3jua/MAQ8lLZL7xARgyLipCKu/0nB64brtPRcy0TEsIgYHBGDO1K1Ooe2mTdf6kKfAUvYoN8ndOhYz9BD5/LMg9WlDisTnnmqD9vt8AEAffp8RIcO9cyf14mqqlqqOidPCO+40/vU14nJk7qVMtQ2V73uItZdJ/nfolPHWgYPmsqkGTVNHtN1nSVsv9UMnnpp03aIsK2IuiKXLGvLmvX7wPqSegILSDqMP9DE/o8DRwOPSNoW2C4tfwa4QtIWETFBUlegT0S81YKYGj0X8AbQX9LmEfFfoLE7vplVXyeu+HEfLrllIhWV8OBtPXjvrXz+ZF0TP/zRM2y3/Yd0q/6EG2+9h5tu2IYHHxjA2eeO4cq/jqS2toLf/3oIIKprPuEXlz5Ofb2YNWsdfnvpkFKH3+Z61izkghMfp6KingrBo2MG8PS4TTh831c56oBx9KhexLUXDefZV/rymxv2BuBzO77L2Nf6sHhJfn+plUvNus2SdUQslXQxSSfwqSQJsSlXAX+T9DrwOkkTCRHxYfp00K2SGqqyPyEZ3Wp1Y2r0XBHxVtq8ca+khcATQK5+E495pBtjHinvmmFzfn3Jbo2W//bSTw+78MH7XTn5hIPaOqRMmTilJ9+++KufKh8+aluGj9q20WMe+M+WPPCfLds6tDaX9VpzMdq0zToi/kTaDr2K7TNJ26wjYhHJkzyN7fcIsEsj5ccXvH4X2DZ9fT3JUIYN2w4peL2qcz1A0nZtZmUkQq5Zm5llXYB7g5iZZZ/nYDQzy7zkBqPbrM3MMi/rTycWw8nazMpaINeszcyyLsI3GM3McsE1azOzjEuaQdxmbWaWeX6C0cws49x1z8wsF9wMYmaWeUlvECdrM7PMc83azCzj/FCMmVlO1JdBb5D8/zYwM2tCQ2+QYpbmSLpO0geSXi0o6yHpIUlvp3+7p+WS9CdJEySNk7RTwTHHpfu/Lem4Yt6Hk7WZlbcQtfWVRS1FuB44cKWy80km6B4IjErXAQ4CBqbLySSzYSGpB3AhsCswBLiwIcE3xcnazMpakDSDFLM0e66Ix4HZKxUfCtyQvr4BOKyg/MZIPAPUSNoIOIBk4u7ZETEHeIhPfwF8ituszazsrcYNxl6SxhasD4uIYc0cs0FETE9fzwA2SF/3ASYX7DclLVtVeZOcrM2srK3mE4wzI2Jwi68VEZKipcc3xc0gZlb2WusG4yq8nzZvkP79IC2fCvQr2K9vWraq8iY5WZtZWWvoZ92GyXoE0NCj4zjg7oLyY9NeIbsB89LmkpHAFyV1T28sfjEta5KbQcysvAXUttITjJJuBYaStG1PIenVcSlwh6STgPeAr6e73wccDEwAFgInAETEbEk/B8ak+10cESvftPwUJ2szK2utOepeRBy1ik37NrJvAKev4jzXAdetzrWdrM2s7PlxczOzjPPYIGZmORFO1mZm2VcOAzk5WZtZWYuAuvr891J2sjazMuc2azOzXHCbtVkTart3KXUImdf1lenN77QWq1i0dI3P4dnNzczyIJJ267xzsjazshZAnSfMNTPLOt9gNDPLBTeDmJnlgHuDmJllXISTtZlZLrjN2swsB+rrnazNzDItkJtBzMzyoAw6gzhZm1mZ8w1GM7OcKIOqtZO1mZW9sq5ZS7qcJr6PIuLMNonIzKwVBeXfG2Rsu0VhZtZWAmjFmrWkc4BvpWd+BTgB2Ai4DegJPA98MyKWSKoCbgR2BmYBR0bEuy257iqTdUTcsFKAXSJiYUsuYmZWSq01NoikPsCZwKCIWCTpDuAbwMHAZRFxm6SrgZOAq9K/cyJiC0nfAH4FHNmSazc7bqCk3SWNB95I17eXdGVLLmZmVhJR5FKcDsA6kjoAXYDpwD7AP9LtNwCHpa8PTddJt+8rqUXV/GIGef0DcABJFZ6IeBnYuyUXMzNrf8lDMcUszYmIqcBvgUkkSXoeSbPH3IioTXebAvRJX/cBJqfH1qb792zJuyhqRO6ImLxSUV1LLmZm1u4Col5FLUAvSWMLlpMLTyWpO0lteQCwMdAVOLA93kYxXfcmS9oDCEkdgbOA19s2LDOzVlR8E8fMiBjcxPb9gHci4kMAScOBPYEaSR3S2nNfYGq6/1SgHzAlbTapJm2lWF3F1KxPAU4nqc5PA3ZI183MckJFLs2aBOwmqUva9rwvMB54FDgi3ec44O709Yh0nXT7IxEtu93ZbM06ImYCx7Tk5GZmmdBKvUEi4llJ/wBeAGqBF4FhwL3AbZJ+kZZdmx5yLfB3SROA2SQ9R1qk2WQtaTPgj8BuJG/5aeCciJjY0ouambWrVnzcPCIuBC5cqXgiMKSRfRcDX2uN6xbTDHILcAdJp++NgTuBW1vj4mZmba7hoZhilgwrJll3iYi/R0RtutwEdG7rwMzMWkvUF7dkWVNjg/RIX94v6XySRymD5Omb+9ohNjOz1pHxWnMxmmqzfp4kOTe8y+8UbAvggrYKysysNamch0iNiAHtGYiZWZtYvUfJM6uo8awlbQsMoqCtOiJubKugzMxaT/ZvHhajmK57FwJDSZL1fcBBwJMkw/6ZmWVfGdSsi+kNcgTJUzozIuIEYHuSRybNzPKhvsglw4ppBlkUEfWSaiV1Az4gedbdMmTw0Pmc8vNpVFYE99/agzv+vEGpQ2p3vXt+zA/OeJLuNYuIgPse3pK77hu0bPv/O+Q1vnPcWI448Ujmf9SZr33lVfb5XPJsV2VF0K/vPL5+0pF8tKCqVG+hzZ31k5cZsucHzJ3TidOP/jwAe+0znaO//Rb9+i/gnBP2ZMIbNQBUVtZz5o/HscVW86msrGfU/X2584YtShh9C7Xy5AOlUkyyHiupBvgrSQ+RBSRPMbY5ST+KiEvW4Pj7gKMjYu5qHnc8MDgizmjptdtTRUVw+iVTueAbmzFzekcuv+9tnhlZzaS3167u8HV1YtiNg5nwTk/W6byUK351Dy+M25hJU2ro3fNjdt5+Gu9/2HXZ/neO2JY7R2wLwG47T+bwQ8aXdaIGePievtxzZ3++d+FLy8rem7guvzxvZ844/5UV9t1r3+l07FTP6cfsTVVVHVfd9hiPPbgxH0zv0s5Rr7ly6A3SbDNIRJwWEXMj4mpgf+C4tDmkPfxoTQ6OiINXN1Hn0VY7LmTau52YMamK2qUVjL67ht0PmFfqsNrd7LldmPBOMlTwosUdmTS1ml49ksmNTjl+DNfctPMqZwwZutc7PPpk+XeAeu2lnnw0v+MKZZPfXY+pk9ZtdP/OneuoqKynU1UdtbUVLPw4p3Nst+7kAyXR1EMxOzW1LSJeaM1AJN1F0rzSmWQsks1IZmN4CXgN+C8wOyL+kO7/S5ImmZeBi4GPgC1IRr86LW26eZekhjxT0rHAuST/ScZFxDclfRn4CdCJZNjCYyLi/dZ8X+2h54ZL+XBap2XrM6d3ZOud1u4Z2DbovYAtBszmjbd7sfvgScyc3YWJ7/VodN+qTrUM3mEqV1y7aztHmW1PjtqIXfd+n5vuHUVV5zr++odBLJjfqfkDrU009TX5uya2Bck0Nq3pxIiYLWkdYAzweeCMiNgBQFJ/YDjwB0kVJKNXDQE+m/4dBLwHPAAczvIpdpC0DUlS3iNN3A3/1z4J7BYRIelbwA+B7zcVZDoY+ckAncnfz8G1QefOS/npuY9y1d92oa6ugqMOf4Xzf7H/KvffbfBkxr+xftk3gayuLbeZS32d+OaX9mXdbkv59V+e5qXnejFjWv7+3aucZzePiC+0ZyDAmZK+mr7uBwxcKZ53Jc2StCOwAfBiRMxKpzN7rmEUQEm3AntRkKxJvljuTId7JSJmp+V9gdslbURSu36nuSAjYhjJkIh0U49M/HCaNaMjvTdesmy910ZLmTm9YxNHlK/Kynp++v3RPPLEZjz13Kb032QOG66/gKt/MwKA3j0XcuWv7+G7F3yJOXPXAWDonu/w6FPl3wSyuoYeMI3nn+lNXV0F8+ZUMX5cd7b4zNz8JescNHEUo6hpvdqapKEkMzDsHhHbk4wH29jdsWuA40mmfr+uoHzl/xTF/qe5HPhzRHyW5HH6XN6Re/OlLvQZsIQN+n1Ch471DD10Ls88uDb2rgy+d+pTTJpazT/v2QaAdyd15+vfOpJjTz+CY08/gg9ndeG0Hx6yLFF36bKEzw56n6fHuIPTyj6csQ7bD04mNanqXMvW285lynuNt21nXjm3WbezapLp2hdK2ppk7GyApZI6RsTSdP1fJO3THYGjC44fImkASTPIkaQ13wKPAP+S9Pu0Nt4jrV1Xs3z6nePIqfo6ccWP+3DJLROpqIQHb+vBe2/l8ntnjWyz9Qfs//mJTHyvO1elNenrbtmJMS/2XeUxew6ZxAsvb8ziT9aOXyI//PmLfHanWXSrWcIN/x7FzcMG8tH8Tpxy7mtU1yzhosvGMPGtbvz0rF255x+bcs7/vsyVtz6GBA/d05d3J3Qr9VtokXLoDaIWzjDTukFIVcBdQH/gTaAGuIjkacmvAC9ExDHpvleTzCR8fro+lOJuMB4H/IBkst8XI+J4SYcClwFzSBL6LhExtNiue93UI3bVvq3yGZSj2HOHUoeQeR0nzSx1CJn2nxm3MO+T99eowbmqX7/oe/Y5Re078dzvP9/MHIwlU8zj5iKZ1muziLhY0ibAhhHxXGsFERGfkCTmlY0GziuIpYKk1r3yzAvzI+KQRs7bv+D1DcANK22/m+VzpRWWXw9cX2T4ZpZ1pa+TrrFi2qyvBHYHjkrXPwKuaLOIVkHSIGACMCoi3m7v65tZPimS3iDFLFlWTJv1rhGxk6QXASJijqR272wZEeNJ+l6vXD6apAZuZta4MqhZF5Osl0qqJH27knqT+SFPzMyWK4cbjMU0g/yJpBfG+ulTg08CLR6vw8ys3a0NXfci4mZJz5MMkyrgsIh4vc0jMzNrDbGW1KzT3h8LgX8DI4CP0zIzs3xoxfGsJdVI+oekNyS9Lml3ST0kPSTp7fRv93RfSfqTpAmSxjU15lJzimkGuRe4J/07CpgI3N/SC5qZtTdFcUuR/gg8EBFbk0zG8jpwPklPtYEkefL8dN+DSIbOGEgyptBVLX0PxTSDfLZwPf1mOK2lFzQzyytJ1cDeJMNeEBFLgCXpA3ZD091uYPkzIocCN0by9OEzaa18o4iYvrrXXu2xQdKhUT2WpJnlR/E3GHtJGluwnLzSmQYAHwJ/k/SipGskdQU2KEjAM0gGmwPoA0wuOH5KWrbainmC8XsFqxXATsC0llzMzKzdrV4Tx8xmHjfvQJIDvxsRz0r6I8ubPJLLJUMut/otzWJq1usVLFUkbdeHtnYgZmZtpvW67k0BpkTEs+n6P0iS9/vpUMukfz9It09lxTlr+7J88LjV0mTNOn0YZr2IOLclJzczKzUBaqXH+CJihqTJkraKiDdJujSPT5fjgEvTvw1jDo0AzpB0G0nz8byWtFdD09N6dYiIWkl7tuTEZmaZ0bqNEt8Fbk6H3ZhIMr5+BXCHpJNIhmr+errvfcDBJOMaLUz3bZGmatbPkVTvX5I0ArgT+LhhY0QMb+lFzczaTSs/FBMRLwGNtWt/arzktBfI6a1x3WLGBulMMpnsPiTfT0r/OlmbWT6UwROMTSXr9dOeIK+yPEk3KIO3bmZrjTLIWE0l60pgXVZM0g3K4K2b2dqiHMYGaSpZT4+Ii9stEjOzthCUxaDOTSXrbE+bYGZWpHKvWXsmWDMrD+WcrCNidnsGYmbWVsq9Zm1mVh6crM3Msm01x6rOLCdrMyt/TtZmZtnnmrWZWR44WZuZ5YCTtZlZxvkGo5lZPrTW5AOl5GRtZuXPNWuzVatYUlfqEDLv3mfvKXUImTbkgHmtch43g5iZZV3xk+FmmpO1mZU/J2szs2wTbgYxM8sF1ec/WztZm1l5K5M264pSB2Bm1tYaRt5rbinqXFKlpBcl3ZOuD5D0rKQJkm6X1Cktr0rXJ6Tb+6/Je3CyNrPyF0UuxTkLeL1g/VfAZRGxBTAHOCktPwmYk5Zflu7XYk7WZlb2WqtmLakv8CXgmnRdwD7AP9JdbgAOS18fmq6Tbt833b9FnKzNrLxF8rh5MUsR/gD8kOXzpfcE5kZEbbo+BeiTvu4DTAZIt89L928RJ2szK3/FN4P0kjS2YDm54RSSDgE+iIjn2zl6wL1BzKzMrWY/65kRMXgV2/YEviLpYKAz0A34I1AjqUNae+4LTE33nwr0A6ZI6gBUA7Na9CZwzdrM1gYRxS1NniIuiIi+EdEf+AbwSEQcAzwKHJHudhxwd/p6RLpOuv2RiGYu0gQnazMre63Zda8R5wHfkzSBpE362rT8WqBnWv494Pw1eQ9uBjGz8tYGD8VExGhgdPp6IjCkkX0WA19rrWs6WZtZ2VMZjNbrZG1mZc8DOZmZZV3Q7M3DPHCyNrOy55q1mVkeOFmbmWWbJx8wM8uDCE8+YGaWC/nP1U7WZlb+3AxiZpZ1AbgZxMwsB/Kfq52szaz8+QajmVkOuM3azCzr2mDUvVJwsjazspY8FJP/bO1kbWblr7jJcDPNydrMyp5r1pYZg4fO55SfT6OyIrj/1h7c8ecNSh1Su+vV62N+cPZ/qKlZDAH3jRzI3fdszef2eI//OWoc/frO46wfHMTbE3oC0KFDHWee9iwDN59NBFx9zWDGvbphid9F6/rdOf149uFu1PSqZdijbwJww6835OmR1UhQ02sp5/5hEj03rOXOK3vzyPAeANTVweS3O3P7K6/SrXsdC+ZVctm5/Xj3jc5I8L3fT2LQ4IWlfGvFiyiLftaZnoNRUo2k09bg+P+08LiLJJ3b0uu2t4qK4PRLpvKTYwbw7aFb8YVD57LJwMWlDqvd1deJv163E98548uc/cMD+fLBb7JJv7m8O6mGn1+6N6++tv4K+x/0xQkAnHrWIVxw4X58+4QXUDl0GyjwxSNn88ubJ65QdsSpH3D1qDe56uE32XW/+dx0WfIF9bXTPuSqh5PyEy+Yzmd3X0C37skUK1f9tA+Dh87n2ife4KqH32STgZ+0+3tZE208B2O7yHSyBmqAFifriNij9ULJrq12XMi0dzsxY1IVtUsrGH13DbsfMK/UYbW72XO6MGFiUmtetKgjk6dU07PHIiZPqWbK1OpP7b9Jv3m8PC5JVPPmdWbBx50YuMWsdo25rX12t49Zr/uKc1p1XW95A+7iRRVInz7u0bu6M/SwOQB8PL+CV57pyoFHzwagY6dg3eqczZPVCrObl1rWk/WlwOaSXpJ0p6TDGjZIulnSoZKOl3S3pNGS3pZ0YcE+CwpenyfpFUkvS7o0Lfu2pDFp2T8ldWnPN9daem64lA+ndVq2PnN6R3pttLSEEZXeBusvYPPNZvPmWz1Xuc/Ed7qz25ApVFTUs8H6Cxi4+Sx698rJT/s19LdLN+SYnQfxyPDuHPuD6StsW7xQjB29HnsdnHzhz5hURXXPWn53ziactv+WXPb9fixemPXUUSBA9cUtWZb1T/x84L8RsQPwZ+B4AEnVwB7Avel+Q4D/B2wHfE3S4MKTSDoIOBTYNSK2B36dbhoeEbukZa8DJ7Xpu7F20bnzUn5y3uP85ZrBLFzUaZX7jXx4cz6c1YXLf3c/p3xrLOPf6E19fSPVzDJ0wvkzuPn58exz+BxGXNd7hW3PPFTNNoM/XtYEUlcHE17pwiHHzuTKh96ic5d6bv/z+o2dNrtcs24/EfEYMFBSb+Ao4J8RUZtufigiZkXEImA4sNdKh+8H/C0iFqbnmp2WbyvpCUmvAMcA2zQXh6STJY2VNHYp2Wi3mzWjI703XrJsvddGS5k5vWMJIyqdysp6/vf8x3n0sf489cwmTe5bX1/BsGsHc/o5X+Jnlwxl3XWXMHXaeu0UaTbs89U5PHnfik1Ej91ds6wJBJJ/T703WsrWOyW/OvY6ZC4TXlmnXeNcY1HkkmG5SdapG4H/AU4ArisoX/ljLvZjvx44IyI+C/wM6NzcARExLCIGR8TgjlQVeZm29eZLXegzYAkb9PuEDh3rGXroXJ558NNttOUvOOe7TzNpcjXDRwxqdu+qTrVUVSXf9ztuP526ugomTa5p4xhLb+rE5b82nh5ZTb8tllc6Pp5fwbhn1mWPA+cvK+uxfi29Nl7C5AnJv/eXnlgvfzcY6+uLWpo9j9RP0qOSxkt6TdJZaXkPSQ+lTbEPSeqelkvSnyRNkDRO0k4tfQ9Z77r3EVBY1bkeeA6YERHjC8r3l9QDWAQcBpy40nkeAn4q6eaIWCipR1q7Xg+YLqkjSc16atu8jbZVXyeu+HEfLrllIhWV8OBtPXjvrWa/d8rONp/5kP2+8A7vvFvDFZclLWTX37QDHTvWceq3x1JdvZiL//dRJr7TnR9ftC81NYv55UWjqK8Xs2Z34TeXld/96P87dVPGPb0u82Z34JidB/HN78/guUe6MeW/VVRUwPp9lnDmr6Ys2/+p+2vYee+P6NxlxcR1+i+m8qszNqV2qdhwkyV8/7JJ7f1WWi5ozYdiaoHvR8QLktYDnpf0EEkT7aiIuFTS+SRNuOcBBwED02VX4Kr072pTZLydRtItJG3R90fEDyQ9ANwVEVen248nSdDVQF/gpoj4WbptQUSsm74+HzgWWALcFxE/knQq8EPgQ+BZYL2IOF7SRcCCiPhtU7F1U4/YVfu29lsuG9rls6UOIfMeuPvvpQ4h04YcMJmxLy9eoxsJ1V03jt0GfaeofR8ce9HzETG4+T0Tku4muZ/2Z2BoREyXtBEwOiK2kvSX9PWt6f5vNuy3uu8j6zVrIuLohtdpb42BwK0r7TYlIg5r5Nh1C15fStK7pHD7VSTfdCsfd9EaBW1m2dIGlVJJ/YEdSSp6GxQk4BlAw1NpfYDJBYdNSctWO1nnps1a0n4kPTYuj4i1rxOxmbVc8b1BejV0IEiXkxs7naR1gX8CZ0fE/BUvFW1yuzLzNesGEfEwsGkj5deTtGWbmX1agOqKzp0zm2sGSe9x/RO4OSKGp8XvS9qooBnkg7R8KtCv4PC+tPDeWG5q1mZmLdZK/awlCbgWeD0ifl+waQRwXPr6OODugvJj014huwHzWtJeDTmqWZuZtUyrPvCyJ/BN4BVJL6VlPyK5H3aHpJOA94Cvp9vuAw4GJgALSbodt4iTtZmVt6DVknVEPEkyn0FjPtU1LG2/Pr01ru1kbWblL+PjfhTDydrMyp4nHzAzy7oA6vJftXayNrMyl/0R9YrhZG1m5c/J2swsB5yszcwyLiiLCXOdrM2szAXU52zOyEY4WZtZeXPN2swsJ9xmbWaWA07WZmZZ537WZmbZF0ARk+FmnZO1mZU/J2szs6wL9wYxM8u8gAjXrM3Mss81azOzHHBvEDOzjIvwDUYzszyIOo8NYmaWcX4oxsws+zyQk5lZTpRB172KUgdgZtaWAoj6KGophqQDJb0paYKk89s2+uVcszaz8hbRajcYJVUCVwD7A1OAMZJGRMT4VrlAE1yzNrPyF/XFLc0bAkyIiIkRsQS4DTi0TWNPKcrgLmmpSPoQeK/UcRToBcwsdRAZ58+oaVn7fDaNiN5rcgJJD5C8r2J0BhYXrA+LiGEF5zoCODAivpWufxPYNSLOWJMYi+FmkDWwpv+IWpuksRExuNRxZJk/o6aV4+cTEQeWOobW4GYQM7PiTQX6Faz3TcvanJO1mVnxxgADJQ2Q1An4BjCiPS7sZpDyMqz5XdZ6/oya5s+nCRFRK+kMYCRQCVwXEa+1x7V9g9HMLAfcDGJmlgNO1mZmOeBkbWaWA07WZmY54N4gOSXp3yRj1DQqIr7SjuFkjqTLafrzObMdw8kcSd9rantE/L69YrHiOFnn12/Tv4cDGwI3petHAe+XJKJsGVvqADJuvVIHYKvHXfdyrrHHg8vxkWGztZ1r1vnXVdJmETERQNIAoGuJY8oMSb2B84BBJIP0ABAR+5QsqAyR1Bk4CdiGFT+fE0sWlDXKNxjz7xxgtKTRkh4DHgXOLm1ImXIz8DowAPgZ8C7JI8OW+DtJM9oBwGMkY118VNKIrFFuBikDkqqArdPVNyLik1LGkyWSno+InSWNi4jt0rIxEbFLqWPLAkkvRsSODZ+PpI7AExGxW6ljsxW5Zp1zkroAPwDOiIiXgU0kHVLisLJkafp3uqQvSdoR6FHKgDKm4fOZK2lboBpYv4Tx2Cq4zTr//gY8D+yerk8F7gTuKVlE2fILSdXA94HLgW4kTUeWGCapO/ATktHj1gV+WtqQrDFuBsm5hp4fDT9n07KXI2L7UsdmZq3HzSD5t0TSOqQPgEjaHHCbdUrSDZJqCta7S7quhCFliqRLGvl8flHCkGwVnKzz70LgAaCfpJuBUcAPSxtSpmwXEXMbViJiDrBj6cLJnIMa+XwOLl04tipus865iHhI0gvAboCAsyIiSxOellqFpO5pEkJSD/zvvlClpKqGHkTpr7SqEsdkjfA/2vLQGZhD8t9zkCQi4vESx5QVvwOelnQnyZfZEcAvSxtSptwMjJL0t3T9BOCGEsZjq+AbjDkn6VfAkcBrQH1aHGv7QE6FJA0CGp5YfCQixpcynqyRdBCwb7r6UESMLGU81jgn65yT9CZJu6xvKhaQ1C0i5qfNHp8SEbPbOyazNeFmkPybCHTEPUBWdgtwCEkf9CBpAmkQwGalCCorJD0ZEXtJ+ogVh5IVyS+zbiUKzVbBNeuck/RPYHuSXiDLEvbaPl6zWblxzTr/RqSLrYKkw4G9SGqQT0TEXaWNKFsk7cTyz+fJiHixxCFZI1yztrIm6UpgC+DWtOhI4L8RcXrposoOST8FvgYMT4sOA+6MCD8YkzFO1jkl6Y6I+LqkV2hk+qqGEebWdpLeAD4T6T90SRXAaxHxmdJGlg3pDertI2Jxur4O8FJEbFXayGxlbgbJr7PTvx5hr2kTgE2A99L1fmmZJaaR9NNfnK5XkQwGZhnjmnVOSXohInaS9PeI+Gap48mqdEKGXYDnSH6BDCGZn3EeeGJhSXeRfD4PkXw++5N8VlPAN6qzxDXr/Ook6Whgj/QG2goiYngjx6yNPNxn0/6VLg1GlygOa4aTdX6dAhwD1ABfXmlbsPyG0dpuO+CmhrFB7FNmA/dGRH2ze1pJOVnnVEQ8CTyZjmd9banjybANgDHpYFfXASPDbX+FjgT+kPbXvy4i3ih1QNY4t1nnnKSuJDOfbBIRJ0saCGwVEZ4pJiVJwBdJBikaDNwBXBsR/y1pYBkhqRtwFMnnEySzD90aEZ44N0M8nnX+XQcsAfZI16cC7iNbIK1Jz0iXWqA78A9Jvy5pYBkREfOBfwC3ARsBXwVekPTdkgZmK3DNOuc8rVfTJJ0FHAvMBK4B7oqIpWl/67cjYvOSBlhikg4Fjid5cOhG4IaI+CCdiHl8RPQvYXhWwG3W+edpvZrWAzg8It4rLIyIes8CD8DRwGWF459L+lVEnCfppBLGZStxM0j+eVqvpm22cqKW9HeAiHi9NCFlysBGJqo4CCAiRpUgHlsF16xzztN6NWubwhVJHYCdSxRLZkg6FTgN2EzSuIJN6wFPlSYqa4rbrHMqHSltlSLihfaKJYskXQD8CFgHWFiwaSkwLCIuKElgGSGpmuRG6/8B5xds+sgTM2STk3VOSXo0fdmZpDvayyQ16+2AsRGxe6liyxJJ/wf8GtiS5LOCpIOI56i0XHEzSE5FxBcAJA0HdoqIV9L1bYGLShha1kwEHgf6Ai+RNBc9zfI5Gc1ywTcY82+rhkQNEBGvAh7+c7kzSQYqei/9gtsRmFvSiMxawDXr/Bsn6RrgpnT9GGBcE/uvbRZHxGJJSKqKiDckeaxmyx0n6/w7ATgVOCtdfxy4qnThZM4USTXAXcBDkuawfGxrs9zwDUZba0j6PFANPBARS0odj9nqcLLOOUl7ktxQ3JSCX0oRsVmpYjKz1udknXPpHIPnAM8DdQ3lETGrZEGZWatzm3X+zYuI+0sdhJm1Ldesc07SpUAlycwwywZwWtufYDQrN07WOVfwJGPDf0iRPKHnhz7MyoibQfJvdCNl/gY2KzNO1vm3oOB1Z+AQwEN/mpUZN4OUGUlVJJPCDi11LGbWejw2SPnpQjJokZmVETeD5JykV1jeRl0J9AYuLl1EZtYW3AySc5I2LVitBd6PiNpSxWNmbcPJ2swsB9xmbWaWA07WZmY54GRtbUZSnaSXJL0q6U5JXdbgXNdLOiJ9fY2kQU3sO1TSHi24xruSehVbvtI+C5ra3sj+F0k6d3VjtLWXk7W1pUURsUNEbAssAU4p3CipRb2RIuJbETG+iV2GAqudrM2yzMna2ssTwBZprfcJSSOA8ZIqJf1G0hhJ4yR9B0CJP0t6U9LDwPoNJ5I0WtLg9PWBkl6Q9LKkUZL6k3wpnJPW6j8nqbekf6bXGJOOAY6knpIelPRaOjWamnsTku6S9Hx6zMkrbbssLR8lqXdatrmkB9JjnpC0dat8mrbWcT9ra3NpDfog4IG0aCdg24h4J0148yJil/Tpy6ckPUgyse1WwCBgA2A8cN1K5+0N/BXYOz1Xj4iYLelqYEFE/Dbd7xbgsoh4UtImwEiSSYUvBJ6MiIslfQk4qYi3c2J6jXWAMZL+mY4d3hUYGxHnSPppeu4zgGHAKRHxtqRdgSvxzOrWAk7W1pbWkfRS+voJ4FqS5onnIuKdtPyLwHYN7dEk024NBPYGbo2IOmCapEcaOf9uwOMN54qI2auIYz9gkLSs4txN0rrpNQ5Pj703nZ+xOWdK+mr6ul8a6yygHrg9Lb8JGJ5eYw/gzoJrVxVxDbNPcbK2trQoInYoLEiT1seFRcB3I2LkSvsd3IpxVAC7RcTiRmIpmqShJIl/94hYKGk0yeBZjYn0unNX/gzMWsJt1lZqI4FTJXUEkLSlpK4ks7QfmbZpbwR8oZFjnwH2ljQgPbZHWv4RsF7Bfg8C321YkbRD+vJx4Oi07CCgezOxVgNz0kS9NUnNvkEF0PDr4GiS5pX5wDuSvpZeQ5K2b+YaZo1ysrZSu4akPfoFSa8CfyH5xfcv4O10243A0ysfGBEfAieTNDm8zPJmiH8DX224wQicCQxOb2COZ3mvlJ+RJPvXSJpDJjUT6wNAB0mvA5eSfFk0+BgYkr6HfVg+PssxwElpfK8BhxbxmZh9ih83NzPLAdeszcxywMnazCwHnKzNzHLAydrMLAecrM3McsDJ2swsB5yszcxywMnazCwH/j/ZlPmvqCWsHwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ConfusionMatrixDisplay(confusion_matrix=cm1, display_labels=[x for x in categories_l1.values()]).plot(xticks_rotation='vertical')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "224619c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6767123287671233 0.8626959247648903\n"
     ]
    }
   ],
   "source": [
    "sens = cm1[1,1]/(cm1[1,1]+cm1[1,2])\n",
    "spec = cm1[2,2]/(cm1[2,2]+cm1[2,1])\n",
    "\n",
    "print(sens,spec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "90d07efc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7f112d7bc0d0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaUAAAFvCAYAAADqodR8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABMsUlEQVR4nO3dd5hU5dnH8e9vl6X3IiKgFBFFVFBUrAFRESzYFTWKMTEaNUZfEzWamGjEmGYv0ZiIvWEvFAsRjaCACFhAeheW3tlyv3+cMzAMW2Zhd8+Z4f5c11ycNufcc4C55ynneWRmOOecc3GQE3UAzjnnXIInJeecc7HhSck551xseFJyzjkXG56UnHPOxUaNqANwmaumallt6kUdRnrq1Yk6gorJsF6xxTVzow4hbS1ar4g6hAqZ/fW6fDNrsaPv79u7ni1bXpTWseMnbRpuZift6LUqgyclt8NqU4/D1SfqMNLT9YCoI6gQFRVHHUKFrNurftQhpO1ng4dGHUKFXNp5zJydeX/+8iLGDm+T1rF5rWY035lrVQZPSs45l9WMIsucHzmelJxzLosZUEzmVAd7UnLOuSxXjJeUnHPOxYBhFGVQxxlPSs45l8UMKPCSknPOubjwNiXnnHOxYODVd8455+IjcyrvPCk551xWM4wir75zzjkXCwZFmZOTPCk551w2M0QBijqMtHlScs65LGZAsZeUnHPOxUWRl5Scc87FgeFJyTnnXIwUmycl55xzMVCM2EzmTMLoScnFRo9eq7nijoXk5hjvPd+Ulx5sGXVI2zn9lO/od8L3SPDeyL157a39ADjt5O84rd80iovF2PGteWLIwRFHCgNO/Y5+fWcEsQ7vyOtv7kv9+pv47W8+pWXLtfzwQ30G3300a9fVjCS+my4cxZFd57JiTR0uGXwOAJf2H8epR37HyrXBTMGPvXkoY77Zkx77zueK0z6nRo0iCgtzefj1w5kwrXW1xvv6cbtTo56Rk2soF/oNXcKcYXWY/GBDVs2owUkvLaHZAQUA5E/K4/PfNwGCSYQPvHo1bU/YWK3xJvOSkttpkkYBN5jZuDKOOQZ4FCgATgbuM7OzK+HavcJrn7Kz50pXTo5x1eAF3Hx+B/IX5fHAu98zZngj5n5fu7pCKNdee66k3wnf88tf96OgMIfBt33I2C/a0KL5Oo48bD5X/upkCgpzadQoui+fbWLtO4Nr/68vBQU53PnHjxj7RWv6953OxEkteemV4zj37K859+yv+feQ7pHE+N6Yzrz6367ccvFH22x/6aMDeOGDg7bZtmptbW78Z1+WrapH+1bL+ftV73LmrRdVZ7gAHP/UUmo32To+QuNOBRx7/zLG3tZkm+MadyrkpFeWkFMDNizJ4Z3TW9K69yJyIvjGzbQ2pZyoA3A75ULgLjPrZmYLKiMhRaVz9/UsnF2TxXNrUViQw6g3GnNE31VRh7WNPdus4rvvm7Npcw2Ki3OY9PVuHHXEXE7pN40Xh+5PQWFQRbJqVfSJdM+2q5k6tRmbNgWxTp6yG0cdMY8jDp/P+x90AOD9DzpwZM/5kcX41YxWrF5fK61jv5/fnGWr6gEwa1ETauUVkVejqCrDS0ujjoU07FC43fYadWxLAiraLBRpThBFlpPWKw7iEUUWktRO0pSk9Rsk/UHSKEl3S/pc0rSwtIOkOpJekPStpNeAOknvPVHSZ5ImSHpZUn1JPwXOBe6Q9Gzy9SQNkvSqpGGSvpf0l7LOFW4/SdJ3kiYAZ1bPXdqq2e4FLF24tRopf1EezVsVVHcYZZo9tzFd91tCgwabqFWzkEMPXkiL5utpvccaunZZwn1/eY+//mkE++ydH3WozJ7TiP33XxrEWquQQ3sEsTZuvJHlK4J/WstX1KZx4+hLdanOPPZrnrz5FW66cBT162zabn+vbrOYNq/5lh8B1Ubw4WXNee/M3fj+xXrlHp7/VU3ePqUl75zWksP+sCKSUhIkZp7NSesVB159F40aZnaYpP7AbcDxwJXAejPbT9KBwAQASc2BW4HjzWydpBuB683sdklHA2+b2SuS2qVcoxvQHdgETJX0ALChpHOFSetx4DhgOvBiaYFLuhy4HKA2dSvjXmSMefMb8dJr+3PXHz5g48YazJzVhOJikZtTTIMGm7n2NyfRudMybvn1aC75+ekQYZXJvPmNeHloFwbf/iEbN9Zgxswg1m0pdiOivT66C0PeOxhD/PSUL7j6zM/487O9tuxvt/tyrhgwlusfOrnaYzvxuSXUbVnMxmU5fPCT5jTsUEDLQzeXenzzgzZzyts/sGpGDT67qSl7HLuR3PQKhpUuk6rvPClF49Xwz/FAu3D5WOB+ADObJGlSuL0n0AX4VEEdQE3gszSu8YGZrQKQ9A2wF9C4lHPtC8wys+/D458hTDypzOwx4DGAhmpaad9pyxbn0WKPrf/Bm7cqIH9RXmWdvtIMf39vhr+/NwCXXvQlS5fVpW3rVXz6WVtATP2+OcUmGjXcxKrV0VbjDR/ZkeEjOwIw6McTyV9Wl5Ura9O0yQaWr6hD0yYbWLUy+qrGZCvWbP2h89an+3H3FcO2rLdovJbBl4/kzqd7szC/YbXHVrdl0JZUu1kxbY/fyLJJNctMSgmNOhZSo24xK6flbekIUZ3MRIFlTu+7eJTXslMh297f5P/9iTqJIsr/YSBgZNhu1M3MupjZZWlcP7neI3GdHT1XlZs6sS6t22+mZdtN1MgrpteAlYwZ0SjqsLaT6MTQovk6juo5j48+bs//xrbloAN+AKD1HqvJq1HMqtUR/SROsiXWFus46sj5fPTfdoz5vA3H95kJwPF9ZvLZ2DZRhridZg3Xb1k+9qBZzFoUdCCoX2cTf7liGI++cRiTZ+5e7XEVrhcFa7VledGntWi8T+kJZu38XIrDpqa1C3JZPTOPem2iaQMLOjrkpPWKAy8pVZ0fgN0kNQPWAqcAw8o4/mPgAuBDSV2BA8PtY4CHJO1tZtMl1QNam9m0HYipxHMB3wHtJHU0sxnAwB04904pLhIP3dKawc/NJCcXRrzQlDnT4vUrHuD3N/6XBg02U1QoHnzsUNatq8nwDzpy/dWf8c/73qKgMIe/3nckUVbdJfzu5tE0aLCJoqIcHnqkB+vW1eTFV7rw2xs/oe8JM1iypB533n10ZPHdNugDundaSKP6Gxl6x7P8+91D6N5pIXu3WQYmFi2vz9+ePxYI2plat1jNoH4TGNRvAgDXP9h/S9fxqrZhWQ4fX90MACsS7U5Zzx7HbGLeyNp88afGbFqey6grmtNk3wKOeyKfJeNr8c3jDcipYZADh962Yptee9VLsenEkA5ZBs1ImGkk/RK4FlgAzARmA70Iu3qH7UXjzKydpDrAf4CDgG8JksVV4XHHAXcDiZ/ft5rZm5KeZNs2pbfNrKukQUAPM7s6jONt4G9mNqqMc50E3AusB0YDHcvrEt5QTe1w9dm5m1RdDjsg6ggqREWZNC0brNurftQhpO1ng4dGHUKFXNp5zHgz67Gj79/7gLr29zf2SevY0zt+tVPXqgxeUqpCZnY/YTtRKfvzCduUzGwDcH4px30IHFrC9kFJy7OBruHyk8CTSftOSVou7VzDCNqWnHNZpsgfnnXOORcHhmLTXpQOT0rOOZfFDCiwzPmqz5xInXPOVZghr75zzjkXH3EZrSEdnpSccy6LmZFRXcIzJ1LnnHM7QBSn+UrrbNJsSZMlTZQ0LtzWVNLIcKzNkZKahNsl6X5J0yVNklTunC6elJxzLosZsNlqpPWqgN7hqDCJZ5puIhjarBPwQbgO0A/oFL4uBx4p78SelJxzLosZotjSe+2EAcCQcHkIcHrS9qcsMAZoLKlVWSfypOScc1muAmPfNZc0LulV0sDMBoyQND5pf0szWxQuLwYS00a3BuYlvXd+uK1U3tHBOeeymAHF6Xd0yE9jmKGjzWyBpN2AkZK+2+Z6ZiZph8ev86TknHNZTZU6n5KZLQj/XBJOSHoY8IOkVma2KKyeWxIevgBom/T2NuG2Unn1nXPOZbFESSmdV3kk1ZPUILEMnAhMAd4ELgkPuwR4I1x+E7g47IXXE1iVVM1XIi8pOedcFqvkSf5aAq+Fk4TWAJ4zs2GSvgBeknQZMAc4Nzz+XaA/wYzW64FLy7uAJyXnnMtylfXwrJnNJJheJ3X7MmC7eWwsmBvpqopcw5OSc85lMYO0H4yNA09KzjmX1TJr5llPSm6XML9Pg6hDqJA2d/0v6hAqpO74qCNI37Ovtok6hGoVdHTwkpJzzrmY8En+nHPOxYIhCiuv912V86TknHNZLJi6wqvvnHPOxYS3KTnnnIuFYJRwb1NyzjkXE5U59l1V86TknHNZzBCFxd7RwTnnXEz4iA7OOediwXvfOeecixXv6OCccy4Wgt53XlJyzjkXE96m5JxzLhYMvPedc865mDCvvnPOORcTPsmfc865WPGSknM7oEev1Vxxx0Jyc4z3nm/KSw+2jDokdq+/lrtO+IBmdTdgBi9/3YVnvjpwy/5Luk/kN0d/xlGPD2Llxjr0bj+La3p+jpkoLM7h7tFHMWFRqwg/QSCO97YsmRRv3GPNtEn+MqfzekxIKpI0UdIUSW9JahxubydpSrjcS5JJOjXpfW9L6hUu15A0WNL34bkmSrqllOv9RNJkSZPCaw4Itz8p6eyUY9emrP9K0kZJjZK29ZK0Krzmt5JuK2F74nV8JdyytOTkGFcNXsCtF7bnZ70603vASvbstLG6Ll+qwmLxl0+O5LRnz2fgy2cy8IApdGyyHAgS1lFt57Nwdf0tx4+d34Yznz+Xs144l9990Js/9hkVSdzJ4npvS5NJ8WZCrMEwQzlpveIgHlFklg1m1s3MugLLgatKOW4+UGKiAf4E7AEcYGbdgGOAvNSDJLUJz3G0mR0I9AQmVSDWgcAXwJkp20eH1+0BXCTp4OTtSa/3K3CtndK5+3oWzq7J4rm1KCzIYdQbjTmi76rqunyp8tfX49ulLQBYX1CTmSuasFv9dQDceMyn/P1/PbGk+vr1BXkQrtfJK8Cs2kPeTlzvbWkyKd5MibUYpfWKA09KO+czoHUp+74CVkk6IXmjpLrAz4BrzGwjgJmtMbM/lHCO3YA1wNrwuLVmNiudwCR1BOoDtxIkp+2Y2TpgPLB3OuesSs12L2Dpwppb1vMX5dG8VUGEEW1vjwar2a9FPpMWt6R3+1n8sLYeU/Obb3dcnw4zeeui53nk1Hf53Qe9I4h0W5lwb5NlUrwZEasF1XfpvOLAk9IOkpQL9AHeLOOwOwmSQrK9gblmtiaNy3wF/ADMkvSf5OrA0F+Tq9tS9p0PvACMBjpL2q6iW1IzgtLX1+GmY1Kq7zqmEeMuoW5eAff2H86fRx9FkYnLe0zgwbGHlnjsBzM7cOozA7nmnZO4pufn1Rypc9tKtCl5UspedcIEsBhoCYws7UAz+xhA0tGlHSPp0jABzJPUNuX9RcBJwNnANOAeSX9IOuTXydVtKaceCLxgZsXAUOCcpH3HSPoSGAH82cwSSSm1+m5GCfFeLmmcpHEFbCrtY1XYssV5tNhj85b15q0KyF+0XY1mJGrkFHFvv+G8M3Uf3p/RgbaNVtO64WpeHfgyIy55hpb11/LK+a/QvO76bd43fuEetGm4msa1N0QUeSDO97YkmRRvpsTqSSm7bQgTwF4EjQeltSklpJaWpgN7SmoAYGb/Cc+3CtjusWsLfG5mdxGUfs4qL0BJBwCdgJGSZofvS67CG21m3c3sEDN7tLzzpcTzmJn1MLMeedSqyFvLNHViXVq330zLtpuokVdMrwErGTOiUflvrHLG7X1GMXNFY4ZMPAiA75c149gnLuXEIRdx4pCL+GFtfc5+4Wzy19dlz0arCH6bwn4tllIzt5iVG2tHFz5xvrcly6R4MyHWxNh3mZKUvEv4DjKz9ZJ+Cbwu6eEyjhsh6Q6gVdL7ngAelPRzM9sYVgXWTH2vpD2A3c1sQripGzAnjfAGAn8IE1niXLMk7ZXu56tuxUXioVtaM/i5meTkwogXmjJnWrRf5gAHt1rMgH2nMTW/KUPPfwmAez87nNFzSr6VJ3ScyWn7TqWwOIeNhTW4YdgJEHEDclzvbWkyKd5MibXIRwnfNZjZl5ImESSB0WUceifwRtL6LcAdwBRJa4ANwBBgYcr78oC/hclpI7AUuCKN0M4H+qdsey3cPraM9x2T0jb1JzN7JY3rVYovPmzIFx82rK7LpWXColbs/8CVZR5z4pCLtiw/MaE7T0zoXtVhVVgc721ZMineuMdqVvnPKYU/pMcBC8zsFEntCdqwmxF0nvqxmW2WVAt4CjgEWAacZ2azyzq3J6UKMrP6KevJnQ+6httGAaOSjnmTpJ/LZlYA3BS+yrrWHOC4UvYNKi02M+tQwr7rk1ZHlbB/FBCvegfnXKWwyq+auxb4Fkhk47uBe8zsBUmPApcBj4R/rjCzvSWdHx53XlknzpwynXPOuR1QuW1K4fOTJwP/CtdF8OM5UasyBDg9XB4QrhPu7xMeXyovKTnnXJarQEmpuaRxSeuPmdljKcfcC/wGaBCuNwNWmllhuD6frc9vtgbmBTFYoaRV4fH5pQXgSck557JYBce+yzezHqXtlHQKsMTMxieGTatsnpSccy6bGRRVXpvSUcBpkvoDtQnalO4DGkuqEZaW2gALwuMXAG2B+ZJqELRbLyvrAt6m5JxzWcwIqu/SeZV7LrObzayNmbUj6M37oZldCHxE8JA/wCVs7W38ZrhOuP9Ds7JHhPSk5JxzWa1aHp69Ebhe0nSCNqMnwu1PAM3C7ddTTo9j8Oo755zLelUxWn3yoy9mNhM4rIRjNrLtEGfl8qTknHNZrgqeU6oynpSccy6LmUFRTCbwS4cnJeecy3JxmGwyXZ6UnHMuy3n1nXPOuVgw0uvuHReelJxzLstlUO2dJyXnnMtq5tV3zjnnYsSKPSk555yLiazofSfpAcqoijSzX1ZJRM5VgbZ/G1f+QTGy4uIjog6hQpo8Pz7qENJmhQVRh1AxO5lQEmPfZYqySkqZ9b/YOefc9gzIhqRkZkOS1yXVNbP1VR+Sc865ypRJ1Xfljj0h6QhJ3wDfhesHSXq4yiNzzjlXOSzNVwykMyDSvUBfwomZzOwr4NgqjMk551ylEVac3isO0up9Z2bzpG0CLqqacJxzzlWqLHxOaZ6kIwGTlAdcC3xbtWE555yrNDGpmktHOtV3VwBXAa2BhUC3cN0551xGUJqv6JVbUjKzfODCaojFOedcVcimkpKkDpLekrRU0hJJb0jqUB3BOeec20kGFCu9VwykU333HPAS0ArYA3gZeL4qg3LOOVd5zNJ7xUE6SamumT1tZoXh6xmgdlUH5pxzrpJk0HNKZY191zRcfE/STcALBGGfB7xbDbE555yrDFnSJXw8QRJKfJqfJ+0z4OaqCso551zlUUxKQekoa+y79tUZiHPOuSoQo6q5dKQ1ooOkrkAXktqSzOypqgrKOedcZYlPz7p0lJuUJN0G9CJISu8C/YBPAE9KzjmXCTKopJRO77uzgT7AYjO7FDgIaFSlUTnnnKs82dD7LskGMyuWVCipIbAEaFvFcbldUI9eq7nijoXk5hjvPd+Ulx5sGXVI27jur7M4/LiVrFyWxxUndgWgfqNCfvvQDFq22cQP82sx+BcdWbs6rVrxSnfr2R9x1L5zWLG2Dhfcex4AnVrlc9MZH1OzRhFFxTn85fWj+WZ+S/ZqsYLfnT2Kzq2X8ujww3h2dLdIYk4o6d4e0385F123gLZ7b+Ta07rw/eR6kcZYkrxaxfx96HTyahWTmwuj32nE039vFXVY28qwSf7SKSmNk9QYeJygR94E4LN0Ti7pdEkmad9wvV24fk3SMQ9KGpS0XiMcPeLPKecaJWmqpK8kfSqpc9L2HknHtZM0JeW990paICknaVtLSW+H5/tG0rtJ798gaWLS6+Kk9x2QtH25pFnh8vtlvVfSbEnNU+KqGcY2XdL34WgZbZL2F4XnmCLpZUl1w+1rU84zSNKDKdsmSnohZVtPSWPDfd9K+kPS+5emxN2lpL/TqpKTY1w1eAG3Xtien/XqTO8BK9mz08bqDKFcI19uzq2X7LPNtvN+sYiJnzbksl4HMvHThpz7i0URRQdvj+/Mr/598jbbruk3hn+934Mf338Oj43swdX9xwCwen1t/v7WUTz78UFRhLqdku7t7Gl1uOPnezNlbIOIoipfwSbxm3M7cuUJ+3LliZ3p0WsN+x68LuqwtiNL71XueaTakj4Pvze/lvTHcHv78LtluqQXJdUMt9cK16eH+9uVd41yk5KZ/cLMVprZo8AJwCVhNV46BhK0Pw1M2rYEuDYRdAlOAKYB5yhlvgzgQjM7CBgC/DWdAMJEdAYwD/hR0q7bgZFmdpCZdQFuSto3w8y6Jb22tJ+Z2eTEduBN4Nfh+vHlvbcEg4EGQGcz6wS8Drya9Lk3hOfoCmwmGBw3nc+8H5ALHCMp+eflEODyMPauBCN1JLyYEvc36VyrsnTuvp6Fs2uyeG4tCgtyGPVGY47ou6o6QyjXlM8bsGbltqWgI05YyftDmwHw/tBmHHniyggiC0yctQerN9TaZpsB9WpvBqB+7c3krw7+OaxYV4dv5+9GYXE6v0urXkn3dt70OsyfWSeiiNIlNq7PBaBGDSM3z2IzMsI2Kq/6bhNwXPg93A04SVJP4G7gHjPbG1gBXBYefxmwItx+T3hcmUr9Fynp4NQX0BSoES6XSVJ94OgwqPOTdi0FPgAuKeWtA4H7gLnAEaUc8zGwd3kxhHoBXwOPsG1ybAXMT6yY2aQ0z1cpwlLPpcB1ZlYUxvAfwr/0Et4ymvQ/80DgaWAEMCBp+27AovBaRdWdeMrSbPcCli7c+jslf1EezVsVRBhReho3L2D5kiDu5UvyaNw8XjHf89ZRXNN/DG/e9DTX9P+Mh4cdHnVIWScnx3h4xHe8OGkKX37cgKlfxq+asbJKShZI1NTkhS8j+M56Jdw+BDg9XB4QrhPu71NCYWMbZVV+/72s2Cj5izPZAGCYmU2TtEzSIYSz1xJky/ck/Tv5DZJqA8cTPKjbmODL9X8lnPtUYHLS+rOSNoTLNYHipH0DCcbqewMYLCnPzAqAh4AXJV0NvA/8x8wWhu/pKGli0jmuMbPR5XzehHTfuzcw18xWp2wfB+xPkLiBoEqToNfjsHBTnZRrNCUotSWcR1Di3Be4hmD8Qgh+qUyVNCo81xAzS9SRnSfp6KRzHGFmG0gh6XLgcoDa1C3hY+3KFJe24i3O7Pk19759JB9N6UCfA6Zzy1mjuOaJU6MOK6sUF4tfnLgv9RoWctsTs9mr8wbmTI1ZCS/9NqXmksYlrT9mZo8lHyApl6ApZ2+C79EZwEozKwwPmU8w1RHhn/MAzKxQ0iqgGZBfWgBlPTzbO91PUYpEiQeCIYoGAg+G554paSxwQcp7TgE+MrMNkoYCv5P0q0RJgq3JZzbBl23ChWY2DoI2IeDtcLkm0B+43szWhNfsC7xtZsMVjHZ+EsEX/pcKnseCsApuBz/3zrw3VXLyGQ08ES5vSL6Ggja5HuFyDyDfzOZKWgD8W1JTM1tuZrdLehY4keDeDyQoSUJQfXd1eQGF/0AfA2ioppX2HbxscR4t9ti8Zb15qwLyF+VV1umrzMr8PJrutpnlS2rSdLfNrMqPV8wnHzKNf7x1FAAfTO7ILWf9N+KIste61TX46tP6HNprTbySUsV61uWbWY+yDgi/j7sp6GvwGsGP30pTJRXKCsbNOw74l6TZwK+Bc9l2FqnBwI0p2wYCx4fvGU+QUZNLZBeG7R2nm9m8NELpS1Dimhye82iSqvDCL+rnzOzHwBfAsWV8pjOSOgGU+ZeWphnAnpJSW3EPIahuhK1tSt3M7Boz20z5BgL7hp93BtAQOCux08xmmNkjBN38D5LUbGc/SGWYOrEurdtvpmXbTdTIK6bXgJWMGRH/Jw/GvN+Y488KKgCOP2sZn41sHG1AKZaursvBHYIKgB4dFzAvP/73NJM0alpIvYZBAaFm7WIOPnYN82bUKuddEaiCLuFmthL4iKCZpXFYowPQBlgQLi8g7K0d7m/E1hqzElVV39WzgafNbMt4eZL+S1JXcjP7TtI3BFVxXyjobn4M0NbMNoXvuZTgS3bkDsYxEPipmT0fnq8eMCtsz+kJjDGz9WFi6EjQjlUiM3uN4FdBpTCzdZKGAP+QdIWZFYU99eoCH+7IOcNOHecCBySqIiX1Bn4HPC7pZOBdMzOgE1AErNz5T7PziovEQ7e0ZvBzM8nJhREvNGXOtHgNRn/T/TM48Ig1NGxSyNNjJvLMPa158eFW/Pbh6fQ9bylLFtTizl90jCy+O85/n4M7LKRxvY28dfPTPDayB3cN/RHXn/opubnGpoJc7not6OvTtP56hlwzlHq1NlNs4vyjJ3P+P85j3abS+h9VrZLu7ZqVNbjyj3No1LSQ2/8zjZnf1OWWiztHEl9pmrYs4IZ755KTY+TkwMdvNWbs+/FL/JU19p2kFkCBma2UVIegmeBuguR0NkGt2CUEzSUQNCtcQtBj+2zgw/D7p1RVlZQGsn0vi6FsP4jrncCX4fIZBAFvStr/BvAXSRX+6REmnpNI6rEWJoJPCBLhnsCDkgoJSoz/MrMvwuq/1Hahf5vZ/Wleuqz3TpKUaO96ieB+/A2YFm7/DjijvL+0MhwDLEhqG4OgU0gXSa2AHwP3SFoPFBKUPIvCdsfUNqVfmFlJ7XlV5osPG/LFhw2r85IV8udflpxwbr6gUmsvdtjvXji+xO2XPHj2dtuWr63LqXf9uKpDSltp9/Z/w5tUcyQVM+vbOlzVN16JskTF5R+SplbAkLBdKQd4yczeDgsYL0j6E8F3eqKp4QngaUnTgeVs2+mtRCrv+y/sKXEh0CFsk9gT2N3MPt/RT+WyQ0M1tcPVJ+ow0qK8aEoAO2rFwEOiDqFCmjw/PuoQ0maF8eohWZ73i18eX147T1lqt2lrba69Lq1jZ/zm/3bqWpUhnTalhwnqDBNtMWsIelw455zLBKb0XjGQTvXd4WZ2sKQvAcxsRRkPvjrnnIubuD2rUIZ0klJBWH9osKWhq/JqKJ1zzlWpTJrkL53qu/sJep3tJulOgmGDBldpVM455ypPNo0SbmbPShpP8FyLgNPN7Nsqj8w559zOM1AG1W2lM8nfnsB64K3kbWZW6jM9zjnnYiQmpaB0pNOm9A7BRxLBdOjtgakE47M555yLuUxqU0qn+u6A5PVwhPBfVFlEzjnndlkVHtHBzCZI8vHvnXMuU2RTSUnS9UmrOcDBwMJSDnfOORcn2dbRgWBm1IRCgjamoVUTjnPOuUqXLSWl8KHZBmZ2QzXF45xzrhKJLOnoIKlGOFPgUdUZkHPOuUqWDUkJ+Jyg/WiipDeBl4F1iZ1m9moVx+acc25nWZaUlJLUJpgp8Di2Pq9kgCcl55zLBFmSlHYLe95NYWsySsigj+icc7u2bOl9lwvUZ9tklOBJyWUU5aYz9nB8NHlxQtQhVMi0xw8o/6CYaPVOVU24XUVeennnz5FB39hl/e0sMrPbqy0S55xzlS9GI4Cno6ykFI9pCJ1zzu2UbOno0KfaonDOOVd1siEpmdny6gzEOedc1ciWkpJzzrlMZ0CW9L5zzjmX4URmdRDwpOScc9nOq++cc87FhbcpOeeciw9PSs4552Ihwyb5y6yxV5xzzlWcpfkqh6S2kj6S9I2kryVdG25vKmmkpO/DP5uE2yXpfknTJU2SdHB51/Ck5JxzWU6W3isNhcD/mVkXoCdwlaQuwE3AB2bWCfggXAfoB3QKX5cDj5R3AU9KzjmX7SqppGRmi8xsQri8BvgWaA0MAIaEhw0BTg+XBwBPWWAM0FhSq7Ku4W1KzjmX5SrQ+665pHFJ64+Z2WMlnlNqB3QHxgItzWxRuGsx0DJcbg3MS3rb/HDbIkrhSck557JZxUYJzzezHuUdJKk+MBT4lZmtlrY+nmtmJu14J3SvvnPOuSwmgt536bzSOp+UR5CQnjWzxAzkPySq5cI/l4TbFwBtk97eJtxWKi8pudjo0Ws1V9yxkNwc473nm/LSgy3Lf1M1uu7umRzWewUrl+VxZb8Dt2w/7eLFnPLjHyguEp9/1Jh/371nhFFudd3dMzn8uJWsXJbHFScFk/D99Oa5HN5nJYUFYuGc2vzj1+1ZtyaarwFtLqbNXVNRoUGRsfbQJiw/Y48t+1s8M5eGo5cx45/dAWg87AcafpwPOaKoQQ1+uGwvCpvXqpZYbx44iqP2n8uKtXX48Z/PAeAnJ43jtCO+Y+XaOgD8851D+eybPamRW8RvzhvNvm2XUmzivleP5Mvpe5R1+qpXSc8pKSgSPQF8a2b/SNr1JnAJ8OfwzzeStl8t6QXgcGBVUjVfiXaJkpKkIkkTJU2R9LKkuiVsf0tS46T37C/pQ0lTw26Ovwv/QpA0SJJJOj7p+NPDbWeH66PC904MX69IuiVpvShp+Zfhe+6VtEBSTtJ5B0laGh73jaSfJW1/MOVzjpLUI2m9WxjTSSnH3RJ255wUnvfw0mKutL+EcuTkGFcNXsCtF7bnZ70603vASvbstLG6Lp+Wka8059ZL991m24E9V9HzhBVcdfIBXHHSgQz9V5ltuNVq5NDm3Dqo8zbbJnzSiJ/3PYAr+x3Aglm1Oe8XZX4/VCnLE/Nv3Ie5d3Rh7u1dqDd5FbWnrwWg1qx15Kwr2ub4TXvVZd5t+zH3T11Ye2gTmr9U5g/uSvXu5525/tH+221/cdQBDPrrWQz661l89k3wY+S0I74D4OK7z+FXD5/M1aePYSdqsyqFzNJ6peEo4MfAcUnfE/0JktEJkr4Hjg/XAd4FZgLTgceBX5R3gV2lpLTBzLoBSHoWuAL4R8r2IcBVwJ2S6hBk+CvNbESYxIYS3NCHwnNOBs4H3g/XBwJfpVz3QjMbl7LtzvB6axPXDtdzgDMIGgV/BHyU9J4XzexqSbsBX0t6M83PPRD4JPxzWHidI4BTgIPNbJOk5kDNcmKucp27r2fh7Josnhv88h31RmOO6LuKud/Xru5QSjXli4bs1nrTNttOvnAJLz26BwWbg98Rq5blRRFaiaZ83pCWKfFOGN1oy/J3X9bj6H4rqjusrSSsdm6wWBSUlkyCYqP5i/NZfEUH6k9YueXwDfs12LK8sWM9GvxvWbWF+tWMVuzedE1ax7bbfQXjpwUlo5Vr67B2Q032bbuUb+fuVpUhlq4SZ541s08ofXzX7ebgMzMj+F5N2y5RUkoxGti7hO2fEfQKAbgA+NTMRgCY2Xrgarb2vU+c5zBJeWGj397AxJ2IqxfwNUE//oElHWBmS4AZwF7lnSws1Z0DDCL4BZP4dm9F0Ji5KTxnvpkt3Im4K0Wz3QtYunBrbsxflEfzVgURRpSe1u030vXQNdzz6hT+8vw37HPg2qhDStuJ5+Yz7r+Nyj+wKhUbe/7uGzr88ivW79+QTR3r0fj9Jazr3piixqUn+IYf57PuwIhjB8465muG3PgKNw8cRYM6wQ+A6QuacXTXOeTmFNOq6Wo6t8mnZZNo/11U4nNKVW6XSkqSahA8zDU5ZXsuQZZPlED2B8YnH2NmM4D6khomNhGUkvoS9MUvqfTybFIR96/lhDcQeB54DTg5bExMjb8D0IGgKAxwXtL5JwLJvWaOBGaFcY8CTg63jwDaSpom6WFJP9qJmHd5ublGg0aFXHfm/vzrrj25+YHpZMJAY+dftZCiQvHh682iDSRHzL2jC7P+cQC1Z66j9tQ11P9iBSuPL71U0eB/y6g1az0r+0Xb5vjap104947zGfSXs1i2ui5Xn/4ZAO+M7czSVfV44v9e49ozP2PK7JYUFUf8VVtJzylVh12l+q5O+KUNQQnniZTtrQkeAhtZwfO+APwSaAT8H/DblP1pVYVJqgn0B643szWSxhIku7fDQ86TdDSwCfi5mS0Pm7deNLOrk84zKum0A8P4EnFeDAw1s7WSDgGOAXoDL0q6ycyeTCdmSZcTPJlNbeqW99HStmxxHi322LxlvXmrAvIXxacqrDT5i2vy6fAmgJg2qT5WDI2aFrJqeXxjP+GspRx+3ApuunBf4jLTTnG9GmzYrwF1v11D3g+baPebKUDQGWKv30xhzl+6AlDn69U0fWsx82/eB8uL9ot+xZqt//7f/Gw//nr5MACKinO4/7Ujt+x79FdvMG9JtKW6TBr7bldJShuS229St4dtRsMJ6j7vB74Bjk0+MCylrE3uk29mn0s6AFhvZtOS++pXUF+gMTA5PEddYANbk9I2yac8YcnvLGCApFsIvnmaSWpgZmvMrIig9DRK0mSC3jJPpnPu8EG6xwAaqmml/baaOrEurdtvpmXbTSxbnEevASv581Xl1lJG7rORTTio5xomjWlE6/YbqJFnrFoe3/9Whxy7krN/vojfnL8fmzbmRhpL7uoCLFcU16uBNhdT9+s1rOjfkln3H7TlmI4//3JLQqo1Zz27PTmHhf/XiaKG0Sf9Zg3Xs2x1kJh+dOAsZi5qAkCtvEIkY+PmPA7tPJ+iIjH7hybRBRqjqrl0xPd/TzUys/VhD7jXJT0MPAv8VtLxZvZ+2PHhfuAvJbz9JmBnu4kNBH5qZs8DSKoHzEr0EtwBfYBJZtY3sSHsyHFGWAorNrPvw13dgDk7HHklKS4SD93SmsHPzSQnF0a80JQ50+LTyQHgxvumc+Dhq2nYpJCnP53A0/e1YcTLLbju7pk88t4kCgvE33/dgbiUPm66bzoH9lwTxPu/L3nm3jacd+VC8moag5+eCgSdHR64tX0k8eWuKqDl47ODX/FmrD2sCeu6NS71+OYvzidnUzGtHpoJQEGzmiz6VUnNw5XvDxd/QPe9F9K4/kZe++OzPPHeIXTfeyGdWi/DEIuX1ecvLwW/Y5s02MA9V7xLsYmlq+px+zO9qyXGMmVQUpKl1w0wo4U93eqXt13SW8BLZvZ0WAJ6gKBjQC7wNHB7+LTyIKBHaulF0pPA22b2SliV1oqgxANB54LkLuRrzax+mHjmA+3MbHXS/leBF4E6pVxruxjCa95AUOIba2aPJu07DbgSuDX8XI0JBlecDlxuZvnlxZyqoZra4dquw00s5dSOV4IrT6b9v5z2z65Rh5C2Vu9k1m/xMS/9enw6oyyUpn6ztta133VpHTv22f/bqWtVhsz629lBJSWkkrab2alJy5MJesSV9L4nKaG6y8wGJS2X+N7Ua4c9+5qWsP/MpNWSrrVdDEnXvLSE499ka2eMI1P3pxOzcy5DZdCPnF0iKTnn3C4rwyb586TknHNZzpOSc865+Mic2jtPSs45l+28S7hzzrl4MLyjg3POufjwkpJzzrlYSEzylyk8KTnnXDYz8+o755xz8eHVd8455+LDk5Jzzrm48JKSc865eDCgOHOykicl55zLct77zjnnXHx47zvnnHNx4W1Kzjnn4sHw3nfOxU3xxp2dsb565dStG3UIFdJp0PioQ0jb8IUTow6hQnJf2rn3C5BX3znnnIsLFXlScs45Fwdefeeccy4+Mmvsu5yoA3DOOVe1ZOm9yj2P9G9JSyRNSdrWVNJISd+HfzYJt0vS/ZKmS5ok6eB0YvWk5Jxz2S4xUnh5r/I9CZyUsu0m4AMz6wR8EK4D9AM6ha/LgUfSuYAnJeecy2YWjOiQzqvcU5l9DCxP2TwAGBIuDwFOT9r+lAXGAI0ltSrvGt6m5Jxz2a5qx75raWaLwuXFQMtwuTUwL+m4+eG2RZTBk5JzzmW5Cjyn1FzSuKT1x8zssXTfbGYm7dz4EZ6UnHMu26WflPLNrEcFz/6DpFZmtiisnlsSbl8AtE06rk24rUzepuScc9nMgOI0XzvmTeCScPkS4I2k7ReHvfB6AquSqvlK5SUl55zLYsIqbZghSc8DvQiq+eYDtwF/Bl6SdBkwBzg3PPxdoD8wHVgPXJrONTwpOedctiuunAmVzGxgKbv6lHCsAVdV9BqelJxzLpslqu8yhCcl55zLcj5KuHPOufjwpOSccy4eMmtAVk9KzjmXzQxPSs7tiB69VnPFHQvJzTHee74pLz3Ysvw3ReT6f8zl8OPXsDK/Bj8/rnPU4ZTourumc9hxK1i5LI8r+3cDoMN+67jmjpnk1SymqEg8dFt7pk1qEG2gKeJ6by8+rAt16heRkwO5NYwHh01jxte1eeCmtmxYl0PLNpu58aE51GtQTGEB3HPDnkyfXIeiQnH8Ocs5/5ol5V+kimTSJH/+8OwOkGSSnklaryFpqaS3w/VB4frEpNdBScvLJc0Kl98P39MtPO9JKdcqCo+bIullSXXD7WtTjhsk6cGUbRMlvZCyraekseG+byX9oYyYu1TibStTTo5x1eAF3Hphe37WqzO9B6xkz07xncJ8xItNueXC9lGHUaaRr+7GrT/Zb5ttl904h2fvb8PVpx3EM/e25bIb50YUXenifG//8vJ0Hnl/Kg8OmwbAvTfsyU9+u5B/fjiVo/qt4pVHdgPg47caU7BJ/PPDqTw4bCrvPt2cxfNqRhd45Y0SXuU8Ke2YdUBXSXXC9RPYfviMF82sW9Lrq8QywZPOvw7Xjw+PHwh8Ev6ZbEN4XFdgM3BFOgFK2g/IBY6RVC9p1xDg8jCOrsBLZcT8TTrXqgydu69n4eyaLJ5bi8KCHEa90Zgj+q6qrstX2JSx9VmzIt4VDVO+aMialdvGaAZ16xcBULdBEct+yIsitDJlwr1NmD+zFgf0XAdA92PX8Mk7jQGQYOP6HIoKYfPGHGrULN5y36udEQzIms4rBjwp7bh3gZPD5YHA8zt6IkkCzgEGASdIql3KoaOBvdM87UDgaWAEwRDyCbsRjtJrZkXVmXjK0mz3ApYu3PpLMn9RHs1bFUQYUXb655/acdlNc3hq9Hh+etNsnvzbXlGHlDlk/HZgR67quw/vPtMMgL322chnwxoBMPrtxixdGCT5Y05ZSe26xQzs1pWLDu3C2VcspWGTiJISaZaSvKSU8V4Azg8TyIHA2JT956VUhdXZ/hRbHAnMMrMZwCi2JrstJNUgmDRrcripTvL5gdtTrx/G+Dzblr7uAaZKek3Sz1MSYLkxS7pc0jhJ4wrYVMZHcnF08gU/8Nid7bj4mEN4bHA7fnXXjKhDyhj/eH06D42Yxp3PzuTNJ5szeUw9rv/HXN4a0oyr+u7DhrU51KgZfLFP/bIeObnGc19O4amx3zL00RYsmuPVd+nwpLSDzGwS0I7gC//dEg5JrQrbUMbpBhIkEMI/k5NInTDpjAPmAk+E2zcknx/4feINknoQjPY7l2AmyO6SmoZx3w70IChBXQAMq0jMZvaYmfUwsx551CrjI1XMssV5tNhj85b15q0KyF8Uv6qlTHf8mUv5dHhTAEa/24zOB60t5x0uIVFyb9y8kKNOWsV3X9Zlz06buOuFmTw0fBq9Tl9Jq72CH2ofvdaYHr3XUCMvOL7LoeuY9lXd6IL3pLTLeBP4GztXdZcLnAX8XtJs4AHgJEmJLlHJyecaM9tc2rmSDAT2Dc83A2gYXgMAM5thZo8QjFd1kKRmOxp/ZZk6sS6t22+mZdtN1MgrpteAlYwZ0SjqsLLOsh9qcsDhqwHodsRqFswurabYJdu4Pof1a3O2LI//bwPa7buRlflB21dxMTx3X0tO+fEyAFq0LmDiJ/W3HP/dhHq03TuijjtmUFSU3isGMqM1Mb7+Daw0s8mSeu3gOfoAk8ysb2KDpCHAGcBTFT2ZpByCUXoPMLOF4bbewO+AxyWdDLwbDpbYCSgCVu5g7JWmuEg8dEtrBj83k5xcGPFCU+ZMi+8X5k0Pz+HAI9bSqGkhz4z7hqf/3pLhz0ee27dx4z3TOPDw1TRsUsjTn4zn6fvacP8tHfj572aTm2ts3pTD/bd0iDrM7cTx3q5YWoM/Xhb0CCwqhN5nrOTQ3mt47V/NeevJ5gAc1W8VJ54fzBR+2qX5/P26PflZr85g4sTzltGhS4S9SWNSCkqHLIOCjQtJa82sfsq2XsANZnaKpEHAX9m2R94vzOx/4bFPAm+b2SuS/gOMNbNHk851GnClmfUr6VolxRBeswfwMnC3mfVM2pcbxtKdoE3pYIKh5AuBW8xseHkxl6Shmtrh2m5wYFcJcupGWNWzA4rXr486hLQNXzgx6hAqJLfV9PE7MPHeFo1qtrQjdy9tcO9tDZt3305dqzJ4UnI7zJNS1fGkVHV2yaTU8vy0jh02//7Ik5JX3znnXLbLoMKHJyXnnMtmiY4OGcKTknPOZTsvKTnnnIsNT0rOOefiIT7j2qXDk5JzzmUzA7PiqKNImycl55zLdl5Scs45Fwve+84551yseEcH55xzcWHF3qbknHMuFuIzLUU6PCk551w2S0yHniE8KTnnXLbLoC7hPsmfc85lMTPDiorSeqVD0kmSpkqaLummyo7XS0rOOZflrJKq78K52R4CTgDmA19IetPMvqmUC+AlJeecy35WnN6rfIcB081sppltBl4ABlRmqD7Jn9thkpYCc6rg1M2B/Co4b1XIpFghs+LNpFih6uLdy8xa7OibJQ0jiC0dtYHkedsfM7PHks51NnCSmf00XP8xcLiZXb2j8aXy6ju3w3bmP0pZJI2LevbLdGVSrJBZ8WZSrBDfeM3spKhjqAivvnPOOZeuBUDbpPU24bZK40nJOedcur4AOklqL6kmcD7wZmVewKvvXBw9Vv4hsZFJsUJmxZtJsULmxVthZlYo6WpgOJAL/NvMvq7Ma3hHB+ecc7Hh1XfOOediw5OSc8652PCk5JxzLjY8KTmXBknHJS23T9l3ZvVH5Fx28qTkIiXpLUlvlvaKOr4kf0taHpqy79bqDCQdkvZNWq6Vsq9n9UdUNkkvJS3fnbJvRPVHVLZMu7+ZxJOSi9rfgL8Ds4ANwOPhay0wI8K4UqmU5ZLW4+C5pOXPUvY9XJ2BpKlT0vIJKfuqZOSQnZRp9zdj+HNKLlJm9l8ASX9PGaLlLUnjIgqrJFbKcknrcZBpSbSse+j3dxfiScnFRT1JHcxsJmxpt6kXcUzJOoTViUpaJlxvX/rbIpNpSbSupO4EtTd1wmWFrzqRRlayTLu/GcMfnnWxIOkkgifiZxJ8Ee0F/NzMhkcaWEjSj8ranyjxxYWkJQTTCgg4L1wmXD/XzFpGFVtJJI2ijC9zM+tdfdGUL9PubybxpORiI2wwTjQgf2dmm6KMJ5WkbsDewNdm9m3E4ZRJ0iVl7TezIdUVSzby+1t1PCm5WJBUF7ieYO6Yn0nqBHQ2s7cjDg0ASb8HLgLGA4cDd5nZ49FGVTGSmgArLYb/6SUdCswzs8Xh+sXAWQTzdf3BzJZHGV864nx/M4n3vnNx8R9gM3BEuL4A+FN04WznPKCbmQ0EDgUujzieMkn6faLbsqRakj4k6M34g6Tjo42uRP8k+PtH0rHAn4GngFXEcKDTDLy/GcOTkouLjmb2F6AAwMzWE69eTJvCmDCzZcT//855wNRw+RKCe9kC+BEwOKqgypCbVBo6j2DG06Fm9juCKtO4ybT7mzG8952Li82S6hA2dkvqCMSpTSm1x13H5Id7zey0aMIq1eakaqS+wAtmVgR8KymO/+9zJdUws0KgD9uWROMYb6bd34zhN8/FxW3AMKCtpGeBo4BBkUa0rQEp638r8aj42CSpK/AD0Bu4IWlf3WhCKtPzwH8l5RM8RD0aQNLeBFV4cZNp9zdjeFJysWBmIyVNAHoSlESuNbP8iMPaorQu35LaEsy+Gasu4cC1wCsEVUr3mNksAEn9gS+jDKwkZnanpA+AVsCIpFJIDnB1dJGVKqPubybx3ncuNiS1Jng+acuPJTP7OLqISiapBXAOMBDYA3jNzG4o+13xIamlmf0QdRzlkVQPOBM438xOjjqedGXK/Y0rLym5WAgH4TwP+BooDjcbEIukJKkBwRfkBcA+wKtAezNrE2lgaZLUmKCL9QXAfgTJNHYk1QROJoizL8Hgt49GGlQaMuX+ZgIvKblYkDQVODBuD8wmSNoAfE4wIvgnZmaSZppZh4hDK1XYcWQAwRdld6ABcDrwsZkVl/HWaifpRIKS54nAR8CLwANm1i7KuMqSSfc3k8S9W6vbdcwE8qIOogw3A7UIRoC+OewdGFuSngOmEYy4/QDQDlhhZqNi+oU5DOgAHG1mF5nZW2wtMcdOBt7fjOHVdy4u1gMTw8buLaUlM/tldCFtZWb3AvdK6kDQseF1YA9JNxK0KU2LMLySdAFWAN8C35pZkaQ4V4scTHBf35c0k2AsudxoQypTpt3fjOHVdy4WShtLLM5jiIVdgi8gGIAzdg94hiMODCRoq8sHOgNd494IL+lIgrjPAr4iSPpxHNUhI+9v3HlScm4XIOkQggR6DjDfzI6MOKRyScoBjifoffeTqOMpSybe37jypOQiJeklMztX0mRKmLrAzA6MIKztSFpDyVMrCDAza1jNIe0QSQKOiVtXe0kHl7XfzCZUVyw7I673N5N4UnKRkrSHmS2UtFdJ+81sTnXHVB5JX5pZ96jjKIukByh7fqJYtNUlSPooafUQgtHYE8zMjqvmkMqUafc3k3hHBxe1twkauf9kZj+OOpg0ZcIvueSp5P9IMIxTbCVP4hcm/VhN6leCjLq/mcRLSi5SkqYQjKp8B/Dr1P1m9mq1B1UOSRPMrMzqpjjJhJJdMr+/uzYvKbmoXQFcCDQGTk3ZZwQjJ0RO0plJq41T1mOZPJP4L8+q5fe3EnlScpEys0+ATySNM7Mnoo6nDMkJ878p67FJnpkqqY1GQBtJ9yfv9zaaXYdX37lYCAffvA7Y08wuj9t06JkmqbeggDoEDydDTHsLlvacWkLcnlfLtPubSTwpuViQ9CJBj6uLzayrpLrA/8ysW7SRBSSdCkxK9AaU9HuChzvnEEyzMSvK+LJBOPr6XsB0M1sZcTguIj72nYuLuE+HfiewFEDSKcBFwE+AN4nhKNaSakv6laQHJV0e99lQJf2UYIT4B4DvJMVtJt9tZNr9zSSelFxcxH06dAsTJQRTWDxhZuPN7F8EE73FzRCgBzAZ6A/8PdpwyvUrYH8zOwI4kmAA3DjLtPubMTy7u7iI+3ToklSfoO2gD8Fo4Qm1owmpTF3M7AAASU8QTLsRZ5vNbCmAmc2UVCvqgMqRafc3Y3hScrEQ9+nQgXuBicBqglGhxwFI6g4sii6sUhUkFsysMBj9JtZSe9xtsx7D3neZdn8zhnd0cJHKpDHPwunadwO+SsyZI6kVkGdmcyMNLoWkImAdW9vlEj3EYtk7LAN73yXuL2zbAy+W9zeTeFJykUoa86w2QR39VwT/sQ8ExoVtDG4XJak2cKqZvRx1LK56eEcHFykz6x2Oc7YIONjMepjZIQTTSy+INrrMJ6m3pKvDV6+o40mHpFxJ/SU9TdDl/ryoY0qHpHqSLpL0TtSxZDJvU3Jx0dnMJidWzGyKpP2iDCiThVWNrwIb2Tri9jlhD8czzCx2CV/SjwjmJOpP0HHgKKB9Uq/H2JFUEziZIO6+wFBi+IhAJvHqOxcLkp4nqKN/Jtx0IVDfzAZGF1Xpwi/9xHTdC82sMMp4Ukl6DXjDzJ5M2X4xcJaZDYgksFJImg/MBR4BXjezNZJmmVn7iEMrkaQTCWadPRH4CHgReMDM2kUZVzbwpORiIWw7uBI4Ntz0MfCImW2MLqqtJN1M0KHh9nB9LrASqAkMMbO7IgxvO5Kmmlnniu6LiqR7gdOBKcBzwBvAZDPrEGFYpZJUDIwGBiVG85A0M67xZhJPSs6lIeyufoyZrQvXvzSz7pJygf+a2dHRRrgtSd+bWacStucA08xs7wjCKlM4a2svghJIf6ARcBnwrpmtjTC07UjqBpxPMP35TOAF4PdmVuJklS593tHBxYKkoySNlDRN0szEK+q4kiUSUui+cFsRQXfguHlb0uPhQLfAlkFvHwXejS6skkm62gIfmdnlQHuC5DQAmB1pcCUws4lmdpOZdSR48LsbkCfpPUmXRxtdZvOSkosFSd8RjBI+HihKbDezZZEFlUTSNIJhcApSttcCppRUKomSpDzgLoJRMRJTyu9JMDzOzamfI2plTewnqY6ZbajumMoiaYSZnZiyLQc4HjjfzH4STWSZz5OSiwVJY83s8KjjKI2kwcDuwNWJ3mBhyeNBYLGZxXKstrC3XaKqboaZrZf0NzO7Icq4Uvlssy7Bk5KLBUl/JujN9ipJA7HGZUSHsO3oTuCnBCUPAW2BJ4Bb49b7riyS5prZnlHHkUxSIVvnJNpmFzEcISGsWi41scd8JuJY86TkYiFpZIfEP8jEl9FxEYVUopSSx/S4VSulQ9I8M2sbdRzJMq3kIWkZQQ/Bkga9M6++23H+8KyLi1ElbIvNLyZJx5aw+dDEQJxm9nH1RlQ2SU1L20W85qnKVHM88VQNT0ouLpK7/NYGTgG+jSiWkvy6hG1GMEZfW7Y+SBsX49k6XXeqWHVyCGXa2Hae2KuIV9+5WAp7tQ03s15Rx1ISSUcBtwJNgDvN7K2IQ8pokl4ys3PD5bvN7Makfdv1dIuapAPNbFK4XMvMNiXt62lmY6KLLrP5c0ouruoCbaIOIpWkPpJGAX8C/mFmPeOYkCRdlLR8VMq+q6s/onIlP8x7Qsq+OM7s+2TS8mcp+x7G7TCvvnOxIGkyW9uQcgm+iG6PLqJtSToZuAVYRdDb7pOIQyrP9WwdR/ABILm79U8IurJnijhW56iU5ZLWXQV4UnJxcUrSciHwQ8y6Wb8FzAeWAb+R9JvknWZ2WiRRlS7TvjTrhrP45gB1wuVEp4w4jphhpSyXtO4qwJOSiwUzm1P+UZHqHXUAFZRpX5qLgX+UsJxYj5vEdO1i26nbBbSOLqzM5x0dnMtCktYD0wm+JDuGy4TrHcysXmnvdeXLtOnbM4knJefSkNLmtR0zO7AawymXpDJHq45byVTSmWXt9xESdh2elJxLQwZ+yceuG3VZwvmJJoYv2LbdK3YjJEh6s6z9MWxjzBjepuRcGkpLOuHI0APZOhJ3XMSxG3VZziSYn+hAguF7njez6WW/JVJHAPOA54GxxLPzSEbykpJzaZDUELiKoBH7TWAkcDXwf8BXMZxePCMHDA1HXh8AnAc0A24xs/9GG9X2wgF6TyD4QXIg8A5BIv060sCygJeUnEvP08AKggclfwr8luDX8elmNjHCuErTiKCbfYkDhhKMxh5HGwmeBVsN7EUw5FTshJM7DgOGhaOPDARGSfqjmWXSM2Cx4yUl59IgabKZHRAu5wKLgD3NbGO0kZUsA+cnOo6g+u4w4H3gBTMbF21UZQuT0ckECakdQQn632a2IMq4Mp0nJefSkPolH/cv/ZKmgpB0cFzmp0oVdnSYBHxCUJLb5ovJzH4ZRVylkfQU0JVgavkXzGxKxCFlDU9KzqVBUhGwLrFKMMrAeuI7CV3X1C/KOCfSTHvuJ0yiiX8PyV+isfz3kEk8KTm3i8iUifQk1Qcws7XlHeuyj48S7tyu449RB1AWSVdKmkvQvX6OpDmSfhF1XK56eVJyLguVNHWFmb0ersdu6gpJtwKnAr3MrJmZNSMYb7BfuM/tIrz6zrkslNx+lAmdNCRNBQ5K7c0oqQ7Bc2D7RBOZq25eUnIuO2Xa1BVWUvd6M9sAFEcQj4uIJyXnslOmTV2xQFKf1I3h80uLIojHRcSr75zLQpk2dYWk/QnGvPsEGB9u7gEcBQzw4Xt2HZ6UnMtCmTaqOYCk2sAFwP7hpm+AZ+M6aoarGp6UnNsFSGoGHAvMNbPx5R0fF4lR2M3s2ahjcdXD25Scy0KS3pbUNVxuBUwBfgI8LelXUcZWEkkNJd0s6UFJJyhwNTATODfq+Fz18ZKSc1lI0tdmtn+4/FtgXzO7WFID4NMYzpT7BltHYe8D7EbQ/nVtTEdhd1XEp65wLjsVJC33AR4HMLM14bhtcdMhaRT2fxHzUdhd1fGk5Fx2mifpGmA+cDDB3D+Jh1HzogysFFuSqJkVSZrvCWnX5NV3zmUhSbsBtwOtgIfMbES4vTdwiJn9Lcr4UiWNwp54sDfWo7C7quNJybldSNjt+lQzeznqWJwrife+cy7LScqV1F/S0wQjcJ8XdUypJNWW9Kuw993lkrxpYRflJSXnspSkHxE8jNof+JxgdIQOZrY+0sBKIOlFgnal0UA/YI6ZXRttVC4KnpScy0KS5gNzgUeA18Ned7PMrH3EoZVI0uSk3nc1gM/jNpK5qx5efedcdnoF2IOgqu5USfWI50CsCcm97wqjDMRFy0tKzmUpSQJ6AQMJqvAaAZcB78ZtqvGk3ncQ9Ljz3ne7KE9Kzu0CJOUBJwHnA33NrHnEITlXIk9Kzu1iJNUJJ89zLna826VzWUjSZMpuQ4rV2HfOJXhJybkslDSfkoB3CNqUtojjfErOgZeUnMtKyUlH0iZPQi5TeJdw55xzseElJeeykKTkB0/rSOrO1sFOMbMJ1R+Vc+XzNiXnspCkj8rYbWZ2XLUF41wFeFJyzjkXG96m5FwWktRJ0uuSpkh6XlLrqGNyLh2elJzLTv8m6Ap+FjABeCDacJxLj1ffOZeFJE00s25J6xN81G2XCbz3nXPZqXZKj7tteuB57zsXV15Sci4Lee87l6k8KTm3i5GUZ2YF5R/pXPXzjg7O7QIU6CPpCWB+1PE4VxpPSs5lMUk9Jd0PzAHeAD4G9o02KudK59V3zmUhSYOBc4C5wPPAa8A4M2sfaWDOlcN73zmXnX4KTAMeAd4ys02S/Beoiz2vvnMuO7UC/gScCsyQ9DRBt3D/IepizavvnMtykmoBpwADgWOAD8zsgmijcq5knpSc24VIagCcYWZPRR2LcyXxorxzWUjSxVHH4NyO8JKSc1lIUmkDsJ4GtDYz/0HqYsmTknNZTpKAC4EbgW+AO81sUrRROVcy/7XkXJYKe9oNAm4AxgBnm9nUSINyrhyelJzLQpKuAq4FPgBOMrPZ0UbkXHq8+s65LCSpGFgCLAWS/5MLKDazgyIJzLlyeEnJuexU0nBCAtoCN1dzLM6lzZOSc1nIzOYklsPJ/S4gGAtvFjA0qricK48nJeeykKR9CEZwGAjkAy8SVNf3jjQw58rhbUrOZaGwTWk0cJmZTQ+3zTSzDtFG5lzZfEBW57LTmcAi4CNJj0vqQ9Cm5FyseUnJuSwmqR4wgKAa7zjgKeA1MxsRaWDOlcKTknO7CElNCDo7nGdmfaKOx7mSeFJyzjkXG96m5JxzLjY8KTnnnIsNT0rOVSFJRZImSpoi6WVJdXfiXE9KOjtc/pekLmUc20vSkTtwjdmSmqe7PeWYtRW81h8k3VDRGF1286TkXNXaYGbdzKwrsBm4InlnOJJ3hZnZT83smzIO6QVUOCk5FzVPSs5Vn9HA3mEpZrSkN4FvJOVK+qukLyRNkvRzCOZBkvSgpKmS3gd2S5xI0ihJPcLlkyRNkPSVpA8ktSNIfteFpbRjJLWQNDS8xheSjgrf20zSCElfS/oXaTzLJOl1SePD91yesu+ecPsHklqE2zpKGha+Z7SkfSvlbrqs5MMMOVcNwhJRP2BYuOlgoKuZzQq/2FeZ2aGSagGfShoBdAc6A12AlgQT9P075bwtgMeBY8NzNTWz5ZIeBdaa2d/C454D7jGzTyTtCQwH9gNuAz4xs9slnQxclsbH+Ul4jTrAF5KGmtkyoB4wzsyuk/T78NxXA48BV5jZ95IOBx4meGbKue14UnKuatWRNDFcHg08QVCt9rmZzQq3nwgcmGgvAhoBnYBjgefNrAhYKOnDEs7fE/g4cS4zW15KHMcDXYJJaAFoKKl+eI0zw/e+I2lFGp/pl5LOCJfbhrEuA4oJxtgDeAZ4NbzGkcDLSdeulcY13C7Kk5JzVWuDmXVL3hB+Oa9L3gRcY2bDU47rX4lx5AA9zWxjCbGkTVIvggR3hJmtlzQKqF3K4RZed2XqPXCuNN6m5Fz0hgNXSsqDYITvcHigj4HzwjanVkBJI3yPAY6V1D58b9Nw+xqgQdJxI4BrEiuSuoWLHxNMa4GkfkCTcmJtBKwIE9K+BCW1hBwgUdq7gKBacDUwS9I54TUkyScYdKXypORc9P5F0F40QdIU4J8EtRivAd+H+54CPkt9o5ktBS4nqCr7iq3VZ28BZyQ6OgC/BHqEHSm+YWsvwD8SJLWvCarx5pYT6zCghqRvgT8TJMWEdcBh4Wc4Drg93H4hcFkY39cEY/E5VyIfZsg551xseEnJOedcbHhScs45FxuelJxzzsWGJyXnnHOx4UnJOedcbHhScs45FxuelJxzzsXG/wNr+KF7jYNY1AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ConfusionMatrixDisplay(confusion_matrix=cm2, display_labels=[x for x in categories_l2.values() if 'not' not in x]).plot(xticks_rotation='vertical')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "42277db6",
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy2 = np.sum((np.eye(5)*cm2))/np.sum(cm2[1:,:])\n",
    "accuracy1 = np.sum((np.eye(3)*cm1))/np.sum(cm1[1:,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b6a64dd9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.8280612244897959, 0.8551724137931035)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy1, accuracy2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "74f645cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "cm_combined = confusion_matrix(y_true=np.array(all_gt_scores_l2)+np.array(all_gt_scores), y_pred=np.array(all_predictions_l2)+np.array(all_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "fd0fc586",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0,   0,  28,  36,  37, 171, 214],\n",
       "       [  0,   0,  43,  37,  78, 132,  75],\n",
       "       [  0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0,   0,  24, 219,   0,   0,   0],\n",
       "       [  0,   0,   7,  28, 161,   7,   2],\n",
       "       [  0,   0,   9,   7,  23, 426,  64],\n",
       "       [  0,   0,   1,   5,  13,  37, 562]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm_combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "285cce60",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['N/A', 'AMF', 'RING SHAPE', 'ANAPHASE-TELOPHASE', 'PROMETAPHASE', 'METAPHASE']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subclass_names"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19c4c038",
   "metadata": {},
   "source": [
    "# Evaluate on TUPAC16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "1f3e7255",
   "metadata": {},
   "outputs": [],
   "source": [
    "db = 'databases_atypia/TUPAC_alternativeLabels_augmented.sqlite'\n",
    "DB = Database().open(db)\n",
    "classes = {x:y for x,y,_,_ in DB.getAllClasses() }\n",
    "databases_directories = [['databases_atypia/TUPAC_alternativeLabels_augmented.sqlite', 'images_mitosis_tupac16/'],\n",
    "                         ]\n",
    "\n",
    "annotations_dict_testset={}\n",
    "\n",
    "class_mapping = { 'Mitose' : -1,\n",
    "                  'Mitosis - AMF OTHER': 0,\n",
    "                  'Mitosis - AMF BIPOLAR ASYMMETRIC CAN ALSO HAVE SEGREGATION': 0,\n",
    "                  'Mitosis - AMF MULTIPOLAR ASYMMETRY CAN ALSO HAVE SEGREGATION': 0,\n",
    "                  'Mitosis - AMF SEGREGATION ONLY':0,\n",
    "                  'Mitosis - RING SHAPE (PROMETAPHASE OR METAPHASE)': 1,\n",
    "                  'Mitosis - NORMAL ANAPHASE-TELOPHASE': 2,\n",
    "                  'Mitosis - NORMAL PROMETAPHASE': 3,\n",
    "                  'Mitosis - NORMAL METAPHASE': 4\n",
    "                }\n",
    "\n",
    "subclass_names = ['N/A','AMF', 'RING SHAPE', 'ANAPHASE-TELOPHASE','PROMETAPHASE','METAPHASE']\n",
    "value_to_class_DB = {y:x for x,y,_,_ in DB.getAllClasses() }\n",
    "value_to_class = {x:y for x,y in enumerate(subclass_names) }\n",
    "\n",
    "skipped_classes=[]\n",
    "\n",
    "for dbfile, folder in databases_directories:\n",
    "    DB.open(dbfile)\n",
    "    for uid, fname in DB.listOfSlides()[0:150]:\n",
    "        fname = folder+fname\n",
    "        DB.loadIntoMemory(uid)\n",
    "        annotations_dict_testset[fname] = {}\n",
    "        for annoI in DB.annotations:\n",
    "            anno = DB.annotations[annoI]\n",
    "            if (value_to_class_DB[anno.agreedClass] not in class_mapping): # skip non-mitotic figures\n",
    "                skipped_classes.append(anno.agreedClass)\n",
    "                continue\n",
    "            cls = class_mapping[value_to_class_DB[anno.agreedClass]]\n",
    "            annotations_dict_testset[fname][annoI] = {'x': [int(anno.x1-25),int(anno.x1+25)], 'y': [int(anno.y1-25), int(anno.y1+25)], 'class': cls}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "787a6c12",
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.Dataset import HierarchicalSublabelsObjectsDataset\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "testset = HierarchicalSublabelsObjectsDataset(annotations_dict=annotations_dict_testset,\n",
    "                                      slide_names=list(annotations_dict_testset.keys()),\n",
    "                                      path_to_slides='./',\n",
    "                                      labels=np.arange(-1,5),\n",
    "                                      crop_size=crop_size,\n",
    "                                      sampling='continuous',\n",
    "                                      pseudo_epoch_length=2000,\n",
    "                                      transformations=None)\n",
    "testset_dl = DataLoader(dataset = testset, batch_size = 8, shuffle = False, \n",
    "                        num_workers = 8, collate_fn = testset.collate_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "50a3faa8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "72baf6f78edf488e8f5e9f25d3ed46a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1197 [00:09<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from tqdm.auto import tqdm\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from src.utils import match_scores_targets,get_number_of_fps\n",
    "model = model.eval().cuda()\n",
    "\n",
    "from torchmetrics.detection.mean_ap import MeanAveragePrecision\n",
    "\n",
    "all_gt_scores = []\n",
    "all_predictions = []\n",
    "all_gt_scores_l2 = []\n",
    "all_predictions_l2 = []\n",
    "all_logits = []\n",
    "all_logits_l2 = []\n",
    "metrics={}\n",
    "metric = MeanAveragePrecision(num_classes=2,iou_thresholds=[0.5], class_metrics=True)\n",
    "overallp = 0\n",
    "overallfp = 0\n",
    "overalltp = 0\n",
    "\n",
    "with torch.no_grad():\n",
    "    for n, (img,target) in enumerate(tqdm(testset_dl)):\n",
    "        pred = model(img.cuda(),[{'boxes': y['boxes'].cuda(), 'labels': y['labels'].cuda(),\n",
    "                                  'sub_labels': y['sub_labels'].cuda(), 'sub_labels_l2': y['sub_labels_l2'].cuda()} for y in target])[0]\n",
    "        pred = [{x:y[x].cpu() for x in y } for y in pred]\n",
    "        for p,t in zip(pred,target):\n",
    "            boxes = p['boxes'].numpy()\n",
    "            scores = p['scores'].numpy()\n",
    "            scl_pred = p['sub_labels'].numpy()\n",
    "            scl_pred_l2 = p['sub_labels_l2'].numpy()\n",
    "            sclogits_pred = p['sub_logits'].numpy()\n",
    "            sclogits_pred_l2 = p['sub_logits_l2'].numpy()\n",
    "            \n",
    "            annos = t['boxes'].numpy()\n",
    "            scl_targets = t['sub_labels'].numpy()\n",
    "            scl_targets_l2 = t['sub_labels_l2'].numpy()\n",
    "            gt_scores, pred_scores,_,_ = match_scores_targets(boxes,scores,scl_pred,annos,scl_targets, radius = 25, det_th = 0.5)\n",
    "            gt_scores_l2, pred_scores_l2,_,_ = match_scores_targets(boxes,scores,scl_pred_l2,annos,scl_targets_l2, radius = 25, det_th = 0.5)\n",
    "            _, pred_logits,_,_ = match_scores_targets(boxes,scores,sclogits_pred,annos,scl_targets, radius = 25, det_th = 0.5)\n",
    "            _, pred_logits_l2,_,_ = match_scores_targets(boxes,scores,sclogits_pred_l2,annos,scl_targets_l2, radius = 25, det_th = 0.5)\n",
    "            \n",
    "            all_gt_scores.extend(gt_scores)\n",
    "            all_predictions.extend(pred_scores)\n",
    "\n",
    "            all_gt_scores_l2.extend(gt_scores_l2)\n",
    "            all_predictions_l2.extend(pred_scores_l2)\n",
    "    \n",
    "            all_logits.extend(pred_logits)\n",
    "            all_logits_l2.extend(pred_logits_l2)\n",
    "            fps = get_number_of_fps(boxes,scores,annos)\n",
    "            overallfp = overallfp + fps\n",
    "\n",
    "            overalltp = overalltp+len(pred_scores)\n",
    "            overallp = overallp+len(annos)\n",
    "            # update metrics\n",
    "            targets=target\n",
    "            for idx, sample in enumerate(targets):\n",
    "                if sample['boxes'].size(0) == 0:\n",
    "                    targets[idx]['boxes'] = torch.zeros((1, 4), dtype=torch.float32)\n",
    "                    # === geändert wegen dataset Änderung === #\n",
    "                    targets[idx]['labels'] = torch.tensor([0],dtype=torch.int64)\n",
    "            metric.update(pred[0:len(targets)], targets)\n",
    "metrics = metric.compute()\n",
    "metrics['precision'] = overalltp / (overalltp + overallfp)\n",
    "metrics['recall'] = overalltp / (overallp)\n",
    "         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "3aad323a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEWCAYAAAAzcgPFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAz50lEQVR4nO3deZgU1dXH8e9xAPcVcEMQjLgMICADo6iIQgSJinFBUARhAFHcApGgQcVdo0ElahDBaIKKBPMKKougYFB2VBAGNQRkFcMqruDAef+oGmzHWXpgeqq75/d5nnmmu+p216nuqjp1b92+Ze6OiIhIstkr6gBEREQKowQlIiJJSQlKRESSkhKUiIgkJSUoERFJSkpQIiKSlFI+QZnZ82Z23x68/ioze6ssY9pdZva5mbUuYt5ZZvZpIpcRx2tbmtnqPY2hmPcfamZ3xDy/zsy+NLNvzKxq+P+4RC2/vJhZrXBdMvbwfaaZWY+yiquEZZV7zGW1vcWz3ZTV/iVlq9gEZWZ7m9kIM1thZl+b2Udmdn4Jr9ntA2BZM7NrzOy9QqbvitHdX3T388o/utJx9+nufmKil2NmzcxsvJltMbNNZjbHzLolerkA7t7b3e8N46gMDAbOc/cD3H1j+H9ZImMws0Fm5maWXWD6Hp0IxXL3leG67CiL94tlZs+Y2Z3hQfkbM/vBzHbEPF+cbDEnWjzbTXntX/nC5Otm9odCpv8iKRdM7GZ2gpn908w2mNlXZrbQzPqW9gTCzBqZ2Xwz+y7836iYsrXDY8NmM1tnZk+aWaWY+W5m38Zsa8Nj5k2Imf6NmW03s49Liq+kGlQlYBVwNnAwMBAYbWa1S3rjimA3NoZKJZeKjpmdDrwDvAscD1QFrgOKPSlJkCOAfYDdOqDGivdzNzMDugCbwv+p6HzgufCgfADQG5iZ/9zd60UcnwS6spvbmZn9CphNcGxu4O4HA5cDWcCBpXifKsBYYCRwKPACMDacXpingf8BRwGNCPLC9QXKNIzZ1nYlVHc/P2b6AcAM4J8lxVhsgnL3b919kLt/7u473f0NYDnQpKQ3LsjM9jKzAWb2XzPbaGajzeywcN4EM7uhQPkFZnZJ+PgkM5scntF/amYdSrv8YuL6WS2ruGWFZ9F/Dc8ivgXOMbPfmNmHZrbVzFaZ2aCY8rXDs4ocM1tJcPDHzHqa2ZKwVpprZqfGhNQoPBv6ysxeMbN9wtf87MzKzGqa2b/MbH34eT4ZTv+Vmb0TTttgZi+a2SFxfhyPAC+4+8PuvsED89290M875vvMX4/fxsw73szeDddjg5m9Ek43M3vMzP4XfmYfm1n9mM/3PjM7AchvbtliZvmfm5vZ8eHjvc3sUTNbaUEz4FAz2zf2szKzP5jZOuBvca7/WQQ7301Ax/wd1cx6AVcB/cOzv9fN7FYze7XA5zHEzJ4IH08zswctqIFuNbOxMdt7/nZRKXx+mJn9zczWWnB2+lo4/VAzeyP8jjeHj48pKngzOwXY4u6FNoslY8wFYtk33AY2m1ku0LTA/KPN7NXwvZeb2U0x8zLM7PaY7XG+mdUM58VuN+3CbfVrM1tjZr8Ppxfcv04OP48tZrbYzC6Kmfe8mT1lZm+G7zPbgqQRFzPbH7gM6APUNbOseF8buhuY4e593f0LAHf/1N2vdPctpXiflgSVkMfdfZu7DwEMOLeI8nWA0e7+g7uvAyYCpT7hsaCCcxbw9xILu3vcfwRntT8AJxVT5nOgdSHTbwZmAccAewPPAC+H87oA78eUzQS2hOX2JzhT6EbwYTYGNgCZYdnngfuKiOUa4L3iYowtE+eyvgLOIEju+xB8yQ3C56cAXwIXh+VrAx5+EfsD+xKc6awh2PmMoKZybExcc4CjgcOAJUDvcF5LYHX4OANYADwWvu8+wJnhvOOBX4efXXXg3wQbYEnfz37ADuCcYr7bXTGEzy8PY90LuAL4FjgqnPcy8MeYzyk/vjbAfOCQcP1PjnnNru8y5rOrFLM8B44PHz8GjAs/pwOB14EHY+LMAx4OP4d949y+RwCjgcrARuDSmHm7YgufHxWu7yHh80oEZ5dNwufTwu+5fvgdvQqMLGzdgDeBVwjOYisDZ4fTqwKXht/NgQRnnK/FxDAN6BHzfED+Z1DYPpCMMReI9SFgevid1gQW8dM2vxfBdnMnUAU4DlgGtAnn3wp8DJxIsF01BKoWst18AZwVPj4UOLWQ/asysBS4PVzWucDXwIkx28JGoFn4Gb4IjCrFcfTqMI4Mgu32L0XtY4V9bsA6oFsJy9hSzN+AsMzvgAkFXvcG0K+I97yW4Fi2H1Aj/H5+W2D/XBvG9y+gdhHvcycwLa7PqhQfamVgCvBMCeU+p/AD4BKgVYEd/MfwCz6QYMc5Npx3P0EzBQQHvukF3usZ4K7CDhyF7Jx5hXxBOyk8QcWzrL+XsP6PA48V2KmPi5k/Cbi5mM+uc8zzPwFDC9mBTgfWE3PwLiaei4EP4/h+aoSxFnfysSuGIuZ/BLQPH/8dGAYcU6DMucBnwGnAXgXm7fouKSZBERyAvgV+FTPvdGB5TJzbgX1KsX3vB2zlp5OLZ4CxhcUWM20C0DN8fAGQGzNvGvBQzPPMMKaM2HUj2A92AofGEWMjYHOBZcQmqOmEB98C+8B7yRpzgbLLgLYxz3vx0zafDawsUP424G/h40/zt71C3jc2Qa0kONAeVNS2TXB2v46Y7ZPghGtQzLYwPGZeO+CTUmxrUwhPGoFOBPty5eL2MX6eoH6M/Zx29w+4gwKJlSDZDiqi/MkEJwl54Wf6PGAx81sQJPRDgCcJEtgvjlEEyf+aeGKMqxefme0F/INgY70hZnrsha+rSnibY4H/C6vMWwgS1g7gCHf/muCMrGNYtlP4QeW/Ljv/deFrrwKOjCd2YJa7HxL7R7CRFhVjSctaFfsCM8s2s6lhs8NXBG3+1Qq8b+xragL/LSbedTGPvwMOKKRMTWCFu+cVnGFmR5jZqLD5YitB+3LBeAqzmeCgc1QcZfOX1cWCjjP5n1X9mGX1J0gkc8Imku4A7v4Owcb7FPA/MxtmZgfFu8xQdYKEMj9m2RPD6fnWu/sPpXjP3xLseOPD5y8C55tZ9aJfwgtA5/BxZ4J9JFbs976C4CSv4HdRE9jk7psLvrmZ7WdBp4cV4Xf5b+AQK+TapwXNuCcRtO0XJ2liLsTRhSw/37HA0QX2zdsJWnXyYypuv8p3KUFCWWFBE/TpRcXh7jsLxFIj5nk8++kvhM2O5/DT8W0sQQvDb8LneQSfeUGVCRITBLW3uPfTYnwDFNz3DiKoLf5MmAMmEtSM9ifYJg4laKUAwN3/7e7bPWhmvJmgSfDkAu9zJsHxdEw8AZaYoMzMCJo+jiBo8sj/kPCfX/h6scg3CawCzi+QLPZx9zXh/JeBTuEGsw8wNeZ17xZ43QHufl08K1hK8SzLC7zmJYKmppoeXKwcSnBgpojXrALibq8uJs5aVvjF/wfC5TVw94MIDkIF4/kFd/8OmEmwA5fIzI4FniU4YakaJv5F+cty93Xu3tPdjyY4Y306/zqAuw9x9yYEZ+gnEDTPlMYG4HugXsz3dLAHF193rVIp37MrwUFmpQXXrf5JcFC4spj3ew04xYJraBfw00EnX82Yx7UIDjAbCpRZBRxmhV8n7EfQZJUdfpctwumFfZ9tgHe85F52yRRzQV8UsvzYZS4vsG8e6O7tYuaXuF+5+1x3bw8cTvBZjC6k2FqgZnhQjo1lTSFlS+tqguPu6+F2tozgeNc1nL8SqGZmu7bl8Bh8LD8l7CmUsJ/az3vMFfy7PSy2mGBbiP1uTqHwjkmHEXwGT3pwvWojwbXddoWUzef88nvvCvzL3b8pLv588dSg/kqQBS909+/jeVOgspntE/NXieDAfX94YMPMqptZ+5jXjCf4Eu4BXok5e3kDOMHMrjazyuFfUzP7WWYuI7uzrAMJziZ/MLNm/HRAK8pw4Pdm1sQCx+d/JqUwh2BnfsjM9g8/4zNi4vkG+MrMalC6g39/4BoLLqZXBTCzhmY2qpCy+xNsgOvDct0IalCEzy+3ny6Obw7L7gw/z2wLupF/S3BNcyelEG4bzwKPmdnh4fJqmFmb4l5nwcXyloVMrwG0IjhgNwr/GhKcHeb3svqS4LpHbBw/EJwJvgTMcfeCNfPOZpZpZvsRbNdjCiYQDy5yTyBI4IeG21z+Qf1AgkS8xYLOCncVs3rtCFohipVkMRc0GrgtfM9jgBtj5s0Bvrag48u+FnSKqG9m+R0phgP3mlndcL86JX8bzmdmVSz43ePB4Yn2Vgrf9mYT1Ir6h+vWErgQKGw/+AULOlcMKmJ2V4JODo1i/i4F2plZ1fD7mA08bGYHmNneBPvwjwTX8CH4TJub2SNmdmS4zOPNbGT+SUNMxaGwvwfC95lG0Ip1kwWdjvJbx94pGLS7byDoIHedmVUKl9MVWBguv54FXdYzwuT6Z4KEviTmc9kX6EDQNBiXkn4HdSzB2W8jYJ3F35w3nmAjzf8bBDxBUNN4y8y+Jviwd/3WxN23EVQfWxPsPPnTvwbOI2j+y78Al3/xu0zt5rKuB+4J1+lOCj8ji13GPwmusb1EUJV+jeDspDRx7iDYYY4nOONaTXD9DIKN/1SCzhxvEnym8b7vDIJrROcCy8xsE8F1pPGFlM0l2AhnEhy8GwDvxxRpCsw2s28IvvebPfgtykEEyWUzwRnhRoLeg6X1B4K27FkWNCVNIThzL5QFTStfE1xIL+hq4CN3fyus+a3zoJfSEH6qbYwAMi1oXnot5rUvEKx7waYywmnPE2xH+xD0DizM1QQHoE8IOi3cEk5/nKBjzQaC/WViEetmBDWoQucXIvKYi3A3wTaxHHgrNr5wm88/gVgevv9wgp+/QPCbudHh67YSfF/7FhH35+E205ugCf9n3H07wf51fricp4Eu7v5JnOtRk5/vCwCY2WkEJ+FPxW5n7j6OYFvuFBa9gqCGt5TgIN8K+E1+k7W7/5fgmmttYLEFlxZeBeZRSPNcUcL1vJjgJGwL0J3gGuz2MN7bzWxCzEsuAdoSnJQuJfj+fxfOO4Kg08xWglphbeCC2Ba3cFlb+Kl1rEQWXrQSSWtm1pmgSfC2Mn7fWgQH6SPdfWvM9GkEPeCGF/XaMoyhGUHTS7M4y0cec7oKa36j3b151LGkg6T+4ahIWXH3kWX9nhZco+hL0BNqa0nlEyyuprQkizntePAbNCWnMqIEJbIbLPix5ZcETVJto4zF3efEUy6ZYhaJh5r4REQkKaX8aOYiIpKeUrKJr1q1al67du2owxARSSnz58/f4O7F/fg8qaRkgqpduzbz5s2LOgwRkZRiZitKLpU81MQnIiJJSQlKRESSkhKUiIgkJSUoERFJSkpQIiKSlBKaoMzsOQtu7b2oiPlmwS2nl1pwm/NTCysnIiIVT6JrUM9T/JAq5wN1w79eBLf2EBERSezvoNz932ZWu5gi7Qluoe4Et004xMyOCu83IyKSlF6avZKxH5XF/Qvj9/WXq2hS/wTuvaRRuS43SlH/ULcGP7/F8+pw2i8SlJn1IqhlUatWrYKzRURKZU+SzOzlmwDIrlOqW7ntlp078vjkrRfJffM5tl/ZB5Sgko+7DyO4eR5ZWVka4VZEgN1PNHuSZLLrHEb7RjW4MjuxJ8sffvgh3bt3Z9FHH3HZZZfx5J8GJHR5ySbqBLWG4O6T+Y4Jp4mIxJV8djfRlFeS2V3z588nOzub6tWr869//Yvf/va3UYdU7qJOUOOAG8xsFMHt37/S9SeRaEVxfaUo8SSfZE80pbVhwwaqVavGqaeeysMPP0z37t059NBDow4rEglNUGb2MtASqGZmqwnu+lkZwN2HAuOBdgT3t/8O6JbIeESkcLFJqTyvr5Qk3ZJPcb7++mtuu+02Ro4cyccff0zNmjXp169f1GFFKtG9+DqVMN+BPomMQaSiKKuL/hUpKSSLSZMm0atXL1atWsWNN95YYWtMBUXdxCcie6Csaj5KStHYsWMHOTk5vPDCC5x00km89957NG/ePOqwkoYSlEiKKKyGpJpPasvIyGDvvfdm4MCBDBw4kL333jvqkJKKEpRIEiuphqSklHq++OILbr75Zm677TYaN27M0KFDMbOow0pKSlAiSaaopKRklNrcneeff56+ffvyww8/cOGFF9K4cWMlp2IoQYmUo9L+rkdJKT0sX76cXr16MWXKFM466yyGDx/OCSecEHVYSU8JSiTBStuRQUkp/YwcOZLZs2fz9NNPc+2117LXXrrTUTws6OmdWrKysnzevHlRhyFSpOKSkpJPxbBkyRI2bNjAWWedxfbt2/nyyy+pWbNmyS9MIDOb7+5ZkQZRCqpBiZQRXTsSgB9//JE//elP3HPPPZx88sl8+OGHVKlSJfLklIqUoERKqajrSEpKMn/+fLp3787ChQu54oorGDJkiDpB7AElKJE4xHMdSUmpYssf3PXwww/ntddeo3379lGHlPKUoEQK0A9ipTTWr19P9erVOfXUU3nkkUfo1q0bhxxySNRhpQV1kpAKZU9u36CkJLG2bt3KgAEDePHFF1m0aFFKXGNSJwmRJBDPdaKiqIYkJRk/fjzXXnsta9eu5ZZbbuGww6If+T0dKUFJ2tB1Ikm0HTt2cM011zBy5EgyMzMZM2YM2dnZUYeVtpSgJOWoF51EJSMjgwMOOIC77rqL2267TYO7JpgSlKQE1Y4kKmvXruXGG2/k9ttvp0mTJjz99NPqOl5OlKAkEqW9uZ5qR1Le3J0RI0bw+9//nm3btnHJJZfQpEkTJadypAQlkRj70Rpyv9hK5lEHxVVeSUnK07Jly+jZsyfvvPMOZ599NsOHD+f444+POqwKRwlKIpN51EG8cu3pUYch8gsvvfQS8+bN45lnnqFHjx4a3DUiSlCScIU155Wm9iRSHhYvXszGjRtp0aIF/fv3p1u3btSoUSPqsCo0nRZIwuU358XKPOog2jfSzi/R2759O/fccw+NGzfmpptuwt2pUqWKklMSUA1K9kg8nR3ya0tqzpNkM2fOHHJycli0aBGdOnXiiSeeUCeIJKIalOyRwmpHBam2JMlo3rx5nH766WzevJlx48bx0ksvUb169ajDkhiqQUmpxdaaVDuSVPPll19yxBFH0KRJEwYPHsw111zDwQcfHHVYUgjVoKTUYmtNqh1Jqvjqq6+49tprqVu3LitXrsTMuPnmm5WckphqUBIX1Zoklb3++uv07t2bdevW0bdvX6pVqxZ1SBIH1aAkLqo1SSrasWMHV155JRdddBFVq1Zl1qxZPPLII+y3335RhyZxUA1KfqaoXnmqNUkqysjI4NBDD+Xuu+9mwIABVKlSJeqQpBRUg5KfKapXnmpNkipWr17NxRdfzPz58wF46qmnuPPOO5WcUpBqUPILqilJKtq5cyfPPvsst956K3l5eXTo0IEmTZpEHZbsAdWgRCTlLV26lFatWtG7d2+aNm3KokWLuPLKK6MOS/ZQwhOUmbU1s0/NbKmZDShkfi0zm2pmH5rZQjNrl+iYRCS9jBo1ig8++IBnn32WKVOmcNxxx0UdkpSBhCYoM8sAngLOBzKBTmaWWaDYQGC0uzcGOgJPJzImEUkPCxcuZNq0aQD079+fJUuW0KNHDw1VlEYSfQ2qGbDU3ZcBmNkooD2QG1PGgfxhrQ8G1iY4JqHk3noiyWrbtm088MADPPDAAzRo0ID58+dTpUoVjj766KhDkzKW6ARVA1gV83w1kF2gzCDgLTO7EdgfaF3YG5lZL6AXQK1aumnd7ojntunqrSfJbNasWeTk5JCbm8vVV1/NY489phpTGkuGXnydgOfd/c9mdjrwDzOr7+47Ywu5+zBgGEBWVpZHEGfKi72Lre5QK6lm3rx5NG/enBo1avDmm2/Srp0uV6e7RCeoNUDNmOfHhNNi5QBtAdx9ppntA1QD/pfg2CokdSGXVLNu3TqOPPJImjRpwpAhQ+jSpQsHHaRm6Iog0b345gJ1zayOmVUh6AQxrkCZlUArADM7GdgHWJ/guEQkyW3ZsoWePXtywgknsGLFCsyMG264QcmpAklognL3POAGYBKwhKC33mIzu8fMLgqL9QN6mtkC4GXgGndXE55IBTZ27FgyMzP529/+xvXXX8/hhx8edUgSgYRfg3L38cD4AtPujHmcC5yR6DgqqsJGIRdJVjt27KBz586MGjWKhg0b8vrrr2s0iApMI0mkOY1CLqkkIyODatWqce+99zJ37lwlpwouGXrxSYKpY4Qks5UrV9KnTx/uuususrKy+Mtf/hJ1SJIkVIMSkUjs3LmTp59+mnr16vHOO++wdOnSqEOSJKMaVBrSdSdJdp999hk9evRg+vTp/PrXv2bYsGHUrl076rAkyagGlYZ03UmS3T//+U8+/vhj/va3vzFp0iQlJymUalBpStedJNksWLCATZs2cc4559C/f39ycnI48sgjow5LkphqUCKSUNu2beOOO+4gKyuLfv364e5UrlxZyUlKpASVRl6avZIrnplZ6C3bRaIwY8YMGjVqxH333cdVV13FlClTNLirxE1NfGkkdjBYXXeSqM2dO5czzzyTmjVrMnHiRNq0aRN1SJJilKDSjK49SdTWrl3L0Ucfves3TV26dOHAAw+MOixJQWriE5EysXnzZrp168aJJ564a3DXPn36KDnJblMNKgXpbriSbP71r3/Rp08f1q9fzx/+8AeOOOKIqEOSNKAElSJ0N1xJRjt27KBjx46MGTOGxo0bM2HCBBo1ahR1WJImlKCSWFFJSXfDlWSRkZHB0UcfzYMPPki/fv2oXLly1CFJGlGCSmK6RbskoxUrVnD99dczaNAgmjZtyhNPPBF1SJKmlKCSnHrlSbLIH9x1wIABmBnLly+nadOmUYclaUy9+ESkRJ988gktWrTgxhtv5Mwzz2TRokV06NAh6rAkzakGJSIlevXVV8nNzeX555+nS5cuGg1CyoVqUCJSqA8++IB33nkHgP79+7NkyRK6du2q5CTlRglKRH7m+++/57bbbqNZs2b0799/1+Cu+m2TlDclKBHZ5b333qNRo0Y89NBDdO3alcmTJ6vGJJHRNagko7vhSlTmzp3LWWedRe3atZk8eTKtW7eOOiSp4FSDSjK6G66UtzVrghOirKws/vrXv/Lxxx8rOUlSUA0qCem3T1IeNm7cSN++fXn11VdZtGgRtWvXpnfv3lGHJbKLEpRIBePuvPrqq/Tp04dNmzYxYMAAjjrqqKjDEvkFJSiRCmTHjh1cccUVvPrqqzRp0oS33nqLhg0bRh2WSKF0DUqkAsnIyKBmzZo8/PDDzJo1S8lJkppqUElAPfckkZYvX851113HPffcQ7NmzXjssceiDkkkLqpBJQH13JNE2LFjB0888QT169dnxowZrFixIuqQREpFNagkoZ57UpZyc3PJyclh1qxZtGvXjqFDh1KzZs2owxIplYTWoMysrZl9amZLzWxAEWU6mFmumS02s5cSGY9IRTF27Fj+85//MHLkSN544w0lJ0lJCatBmVkG8BTwa2A1MNfMxrl7bkyZusBtwBnuvtnMDk9UPCLpbv78+WzevJnWrVvz+9//nh49elC9evWowxLZbYls4msGLHX3ZQBmNgpoD+TGlOkJPOXumwHc/X8JjCepqGOElJXvv/+eQYMG8eijj9K4cWPmzp1L5cqVlZwk5SWyia8GsCrm+epwWqwTgBPM7H0zm2VmbYt6MzPrZWbzzGze+vXrExBu+VLHCCkL7777Lg0bNuRPf/oTOTk5TJkyRYO7StqIupNEJaAu0BI4Bvi3mTVw9y0FC7r7MGAYQFZWlpdjjGWmsFqTOkbI7pozZw4tW7bkuOOO4+233+bcc8+NOiSRMpXIGtQaIPbK7DHhtFirgXHu/qO7Lwc+I0hYaUm1JikLq1YFDRNNmzblmWeeYeHChUpOkpYSmaDmAnXNrI6ZVQE6AuMKlHmNoPaEmVUjaPJblsCYIpdfa3rl2tO5MrtW1OFICtmwYQOdO3fmpJNO4vPPP8fM6NWrF/vvv3/UoYkkRMISlLvnATcAk4AlwGh3X2xm95jZRWGxScBGM8sFpgK3uvvGRMUkkorcnVGjRnHyySczevRobr31Vo4++uiowxJJuIReg3L38cD4AtPujHnsQN/wT0QKyMvL47LLLmPs2LE0bdqUESNG0KBBg6jDEikXGupIJIlVqlSJX/3qVzz66KPMnDlTyUkqFCUokSTz3//+l/POO4/Zs2cD8Oc//5l+/fqRkZERcWQi5SvqbuZpTz/IlXjlD+46cOBAKleuzOrVq8nOzo46LJHIqAaVYOpaLvFYtGgRzZs3p1+/frRq1YrFixdz6aWXRh2WSKRUg0oA/SBXSuuNN95g2bJlvPTSS3Ts2FGjQYigGlRCqNYk8ZgzZw6TJ08GoF+/fixZsoROnTopOYmEdqsGZWZ7AZ3c/cUyjidtqNYkRfnuu++44447ePzxx2nSpAmtW7emcuXKVKtWLerQRJJKsTUoMzvIzG4zsyfN7DwL3Egw2kOH8glRJH1MnTqVBg0aMHjwYHr16qXBXUWKUVIN6h/AZmAm0AO4HTDgYnf/KLGhiaSXOXPmcO6553L88cczbdo0zj777KhDEklqJSWo49y9AYCZDQe+AGq5+w8Jj0wkTaxYsYJjjz2Wpk2bMnz4cDp16sR+++0XdVgiSa+kThI/5j9w9x3AaiUnkfisX7+eTp06kZmZyfLlyzEzcnJylJxE4lRSDaqhmW0laNYD2Dfmubu7fnUa0g9yJZ+78/LLL3PTTTexdetWBg4cSI0a6skpUlrFJih319gqxYhNSrOXbwIgu85h6lpegeXl5XHJJZfw+uuvk52dzYgRI6hXr17UYYmkpGITlJntA/QGjgcWAs+Ft9EQfvq9U+ZRB5Fd5zDaN6qhezxVcJUqVeKEE07gscce48Ybb9T4eSJ7oKQmvhcIrkNNB9oB9YCbEx1UKtHvneQ///kPvXv35v777+e0007j0UcfjTokkbRQUoLKjOnFNwKYk/iQRFJDXl4ejz32GHfeeSd77703a9eujTokkbRSUoKK7cWXpx8UigQWLlxITk4O8+bNo3379jz99NO6y61IGSspQTUKe+1B0HNPvfhEgAkTJrBy5UpGjx7NZZddptEgRBKgpAS1wN0bl0skIklu1qxZbN26lfPOO49+/frRo0cPqlatGnVYImmrpB/qerlEIZLEvv32W373u9/RvHlzBg4ciLtTqVIlJSeRBCupBnW4mfUtaqa7Dy7jeFJC/u+f9IPc9Pf222/Ts2dPli9fzvXXX8+DDz6o5jyRclJSgsoADuCnkSSEn//+ST/ITV+zZ8+mdevW1K1bl3fffZcWLVpEHZJIhVJSgvrC3e8pl0hSjH7/lL6WL19OnTp1aNasGc899xwdO3Zk3333jToskQqnpGtQqjlJhfHll1/SoUMH6tWrt2tw127duik5iUSkpATVqlyiEImQu/P3v/+dk08+mbFjxzJw4ECOOeaYqMMSqfBKGix2U3kFIhKFvLw82rdvz/jx42nevDkjRozgpJNOijosEaHka1AiacndMTMqVapE/fr1adu2Lddff70GdxVJIiU18Ymknc8++4xzzjmHWbNmAfDwww9r5HGRJKQEJRVGXl4eDz/8MKeccgoLFixg3bp1UYckIsVQE59UCAsWLKB79+588MEHXHLJJTz11FMceeSRUYclIsVQgoqTbume2iZNmsSaNWsYM2YMl156adThiEgcEt7EZ2ZtzexTM1tqZgOKKXepmbmZZSU6pt2RP3oEoBEkUsT777/PxIkTAejbty+5ublKTiIpJKE1KDPLAJ4Cfg2sBuaa2Th3zy1Q7kCCO/XOTmQ8e0qjR6SGb775httvv50nn3ySpk2b0qZNGypVqsRhhx0WdWgiUgqJrkE1A5a6+zJ33w6MAtoXUu5e4GHghwTHI2nurbfeon79+jz55JPccMMNvP322xrcVSRFJTpB1QBWxTxfHU7bxcxOBWq6+5vFvZGZ9TKzeWY2b/369WUfqaS82bNn06ZNG/bZZx+mT5/OkCFDOOCAA6IOS0R2U6SdJMxsL2AwcE1JZd19GDAMICsrq1zuU6WOEalh2bJlHHfccTRr1owXXniBDh06sM8++0QdlojsoUTXoNYANWOeHxNOy3cgUB+YZmafA6cB45Klo4Q6RiS3devWcdlll1G/fv1dg7t26dJFyUkkTSS6BjUXqGtmdQgSU0fgyvyZ7v4VUC3/uZlNA37v7vMSHFfc1DEi+bg7L7zwAn379uW7775j0KBBGtxVJA0lNEG5e56Z3QBMIrj54XPuvtjM7gHmufu4RC5f0k9eXh4XXnghEydO5Mwzz2T48OGceOKJUYclIgmQ8GtQ7j4eGF9g2p1FlG2Z6HgkNcUO7tqwYUMuuOACrrvuOvbaS6N1iaQr7d2S9JYsWUKLFi2YMWMGAA899BB9+vRRchJJc9rDJWn9+OOPPPDAAzRq1Ijc3Fw2bNgQdUgiUo40Fp8kpQ8++IDu3buzYMECOnTowJAhQzjiiCOiDktEypESlCSlt99+my+//JL/+7//4+KLL446HBGJgJr4JGlMnz6dCRMmAPC73/2OJUuWKDmJVGBKUBK5r7/+mj59+tCiRQvuvvtu3J1KlSpxyCGHRB2aiERITXwFaHij8jVhwgSuvfZaVq9ezS233MJ9992nwV1FBFCC+oX84Y0yjzpIwxsl2KxZs2jXrh0nn3wy77//PqefrhE7ROQnSlCF0PBGiePuLF26lLp165Kdnc0//vEPLr/8cvbee++oQxORJKNrUFJu1q5dyyWXXELDhg13De7auXNnJScRKZQSlCScuzNixAgyMzOZOHEid999NzVr1iz5hSJSoamJTxIqLy+Pdu3aMXnyZFq0aMHw4cOpW7du1GGJSApQgpKEiB3ctWnTplxyySX06tVL4+eJSNx0tJAyl5uby5lnnrlrcNf777+f3r17KzmJSKnoiCFlZvv27dx77700btyYTz/9lE2bNkUdkoikMDXxSZmYN28eOTk5LFy4kI4dOzJkyBCqV68edVgiksKUoKRMTJs2jQ0bNjB27FguuuiiqMMRkTSgJj7Zbe+++y7jxwc3S77lllvIzc1VchKRMqMEJaW2detWrrvuOlq2bMm99967a3DXgw8+OOrQRCSNKEFJqbz55pvUq1ePYcOG0bdvX6ZMmaLBXUUkIXQNSuI2a9YsLrjgAurVq8eYMWPIzs6OOiQRSWOqQUmx3J3PPvsMgOzsbF566SU++OADJScRSTglKCnSmjVruPjii2nYsCHLli3DzOjUqRNVqlSJOjQRqQCUoOQX3J1nn32WzMxMJk+ezP3338+xxx4bdVgiUsHoGpT8TF5eHueffz5TpkzhnHPO4dlnn+VXv/pV1GGJSAWkBIVu8w4/H9w1OzubDh060KNHD/XQE5HIVNgEFZuUZi8PxozLrnNYhbzN+6JFi+jZsyePPvooZ5xxBvfdd1/UIYmIVNwENfajNbtqS9l1DqN9oxpcmV0r6rDK1fbt23nggQd44IEHOPjgg9myZUvUIYmI7FJhExRA5lEH8cq1p0cdRiTmzJlD9+7dWbx4MVdddRWPP/441apVizosEZFdKnSCqsimT5/OV199xRtvvMFvfvObqMMREfmFhHYzN7O2ZvapmS01swGFzO9rZrlmttDM3jYz9WVOoKlTp/LGG28AweCuixcvVnISkaSVsARlZhnAU8D5QCbQycwyCxT7EMhy91OAMcCfEhVPRbZlyxZ69erFueeey4MPPoi7k5GRwUEHVbzeiiKSOhJZg2oGLHX3Ze6+HRgFtI8t4O5T3f278Oks4JgExlMhjRs3jnr16jFixAhuvfVWJk+erK7jIpISEnkNqgawKub5aqC4AdxygAlFzTSzXkAvgFq1KlZvu901c+ZM2rdvT4MGDRg7dixZWVlRhyQiErekGOrIzDoDWcAjRZVx92HunuXuWbqVeNHcnU8++QSA0047jVGjRjFv3jwlJxFJOYlMUGuAmjHPjwmn/YyZtQb+CFzk7tsSGE/aW7VqFRdeeCGNGzdm+fLlmBlXXHGFBncVkZSUyAQ1F6hrZnXMrArQERgXW8DMGgPPECSn/yUwlrS2c+dOhg4dSr169Zg6dSoPPfSQmkFFJOUl7BqUu+eZ2Q3AJCADeM7dF5vZPcA8dx9H0KR3APDP8ML9Sne/KFExpaO8vDzOO+88pk6dSuvWrRk2bBh16tSJOiwRkT2W0B/quvt4YHyBaXfGPG6dyOWns9jBXc866yw6d+5Mt27d1ENPRNJGUnSSkNJZsGAB2dnZvPfeewDcfffddO/eXclJRNKKElQK2bZtG3fccQdZWVmsWLGCr7/+OuqQREQSRmPxpYhZs2bRvXt3lixZQpcuXRg8eDBVq1aNOiwRkYSpUAkqlW9MOGPGDL755hvGjx/P+eefH3U4IiIJV6Ga+PLvAQWkxI0J3377bV5//XUAbr75ZhYvXqzkJCIVRoWqQUFq3ANqy5Yt9OvXj+eee44zzjiDCy64gIyMDA488MCoQxMRKTcVqgaVCl577TUyMzN54YUXGDBggAZ3FZEKq8LVoJLZzJkz+e1vf0vDhg15/fXXadKkSdQhiYhERjWoiLk7ubm5AJx++umMHj2auXPnKjmJSIWnBBWhlStX0q5dO0499VSWLVsGwOWXX07lypUjjkxEJHpKUBHYuXMnTz31FPXq1WP69Ok8+uij1K5dO+qwRESSiq5BlbO8vDxat27Nu+++y3nnncczzzyj5CQiUgglqHISO7hry5Yt6datG126dFEPPRGRIqiJrxx89NFHNG3alOnTpwMwaNAgunbtquQkIlIMJagE+uGHH/jjH/9IVlYWq1ev5ttvv406JBGRlKEmvgSZMWMGOTk5fPLJJ3Tt2pXBgwdz2GGHRR2WiEjKUIJKkNmzZ/Pdd98xceJE2rRpE3U4IiIpR018ZWjSpEmMHTsWgJtuuonFixcrOYmI7CYlqDKwadMmrrnmGtq2bcuf//xn3J2MjAwOOOCAqEMTEUlZSlB76NVXXyUzM5ORI0fyxz/+kbfeeku980REyoCuQe2BGTNmcNlll9G4cWMmTpxIo0aNog5JRCRtqAZVSu7OokWLAGjevDljxoxhzpw5Sk4iImVMCaoUPv/8c9q0aUNWVtauwV0vvfRSKlVSRVREpKwpQcVh586d/OUvf6F+/frMnDmTwYMHa/w8EZEE06l/CfLy8jj33HOZPn06bdu2ZejQoRx77LFRhyUikvaUoIqwc+dO9tprLypVqkSrVq3o2bMnnTt3Vg89EZFyoia+QsyfP58mTZrw7rvvAnDXXXdx9dVXKzmJiJQjJagY33//PQMGDCA7O5t169axbdu2qEMSEamw1MQXeu+99+jevTv/+c9/yMnJ4ZFHHuHQQw+NOiwRkQpLCSo0f/588vLymDJlCq1atYo6HBGRCq9CN/FNmDCB1157DYAbbriBjz/+WMlJRCRJJDxBmVlbM/vUzJaa2YBC5u9tZq+E82ebWe1Ex7Rx40a6dOlCu3btGDx48K7BXffff/9EL1pEROKU0ARlZhnAU8D5QCbQycwyCxTLATa7+/HAY8DDiYrH3Vk1/x0yMzN5+eWXueOOO5g8ebJ654mIJKFEX4NqBix192UAZjYKaA/kxpRpDwwKH48BnjQzc3cv62A2LlvEzGcH0qRJEyZPnswpp5xS1osQEZEykugEVQNYFfN8NZBdVBl3zzOzr4CqwIbYQmbWC+gFUKtWrd0KpsVZZ1C98hOMvPt6jZ8nIpLkUuYo7e7DgGEAWVlZu1W7uuvCenBhvTKNS0REEiPRnSTWADVjnh8TTiu0jJlVAg4GNiY4LhERSXKJTlBzgbpmVsfMqgAdgXEFyowDuoaPLwPeScT1JxERSS0JbeILryndAEwCMoDn3H2xmd0DzHP3ccAI4B9mthTYRJDERESkgkv4NSh3Hw+MLzDtzpjHPwCXJzoOERFJLRV6JAkREUleSlAiIpKUlKBERCQpKUGJiEhSslTs0W1m64EVu/nyahQYpaIC0DpXDBVtnSva+sKer/Ox7l69rIJJtJRMUHvCzOa5e1bUcZQnrXPFUNHWuaKtL1S8dVYTn4iIJCUlKBERSUoVMUENizqACGidK4aKts4VbX2hgq1zhbsGJSIiqaEi1qBERCQFKEGJiEhSStsEZWZtzexTM1tqZgMKmb+3mb0Szp9tZrUjCLNMxbHOfc0s18wWmtnbZnZsFHGWlZLWN6bcpWbmZpby3XPjWWcz6xB+z4vN7KXyjrGsxbFd1zKzqWb2Ybhtt4sizrJkZs+Z2f/MbFER883MhoSfyUIzO7W8YywX7p52fwS39vgvcBxQBVgAZBYocz0wNHzcEXgl6rjLYZ3PAfYLH1+Xyuscz/qG5Q4E/g3MArKijrscvuO6wIfAoeHzw6OOuxzWeRhwXfg4E/g86rjLYL1bAKcCi4qY3w6YABhwGjA76pgT8ZeuNahmwFJ3X+bu24FRQPsCZdoDL4SPxwCtzMzKMcayVuI6u/tUd/8ufDqL4A7HqSqe7xjgXuBh4IfyDC5B4lnnnsBT7r4ZwN3/V84xlrV41tmBg8LHBwNryzG+hHD3fxPcH68o7YG/e2AWcIiZHVU+0ZWfdE1QNYBVMc9Xh9MKLePuecBXQNVyiS4x4lnnWDkEZ2CpqsT1DZs9arr7m+UZWALF8x2fAJxgZu+b2Swza1tu0SVGPOs8COhsZqsJ7j13Y/mEFqnS7u8pKeE3LJTkY2adgSzg7KhjSRQz2wsYDFwTcSjlrRJBM19Lghryv82sgbtviTKoBOsEPO/ufzaz0wnu0F3f3XdGHZjsmXStQa0BasY8PyacVmgZM6tE0DSwsVyiS4x41hkzaw38EbjI3beVU2yJUNL6HgjUB6aZ2ecE7fTjUryjRDzf8WpgnLv/6O7Lgc8IElaqimedc4DRAO4+E9iHYFDVdBbX/p7q0jVBzQXqmlkdM6tC0AliXIEy44Cu4ePLgHc8vPqYokpcZzNrDDxDkJxS/dpEsevr7l+5ezV3r+3utQmuuV3k7vOiCbdMxLNdv0ZQe8LMqhE0+S0rxxjLWjzrvBJoBWBmJxMkqPXlGmX5Gwd0CXvznQZ85e5fRB1UWUvLJj53zzOzG4BJBL2AnnP3xWZ2DzDP3ccBIwiaApYSXIzsGF3Eey7OdX4EOAD4Z9gfZKW7XxRZ0HsgzvVNK3Gu8yTgPDPLBXYAt7p7yrYMxLnO/YBnzex3BB0mrknxk03M7GWCE41q4bW1u4DKAO4+lOBaWztgKfAd0C2aSBNLQx2JiEhSStcmPhERSXFKUCIikpSUoEREJCkpQYmISFJSghIRkaSkBCVShJJGlA7L7DCzj2L+aptZSzP7Kny+xMzuCsvGTv/EzB4tv7URST1KUCJFex4oaSy77929Uczf5+H06e7eiGBIqc4xt0PIn94YuMDMzij7sEXSgxKUSBHiGFE6nvf4FpgPHF9g+vfAR6ThAJ8iZUUJSmTP7BvTvPd/BWeaWVWCcQAXF5h+KMEYef8unzBFUk9aDnUkUo6+D5vsCjrLzD4EdgIPhcPztAynLyBITo+7+7pyi1QkxagGJRInM6sZU1vqXULx6e7e2N2bhGOnxU5vCNQDcsysUcICFklxqkGJxMndVwGNyui9lpvZQ8AfCO5nJCIFqAYlUoRwROmZwIlmttrMcsp4EUOBFmZWu4zfVyQtaDRzERFJSqpBiYhIUlKCEhGRpKQEJSIiSUkJSkREkpISlIiIJCUlKBERSUpKUCIikpT+H0FP2BjOeMApAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sel = np.stack(all_gt_scores)>=0\n",
    "from sklearn.metrics import roc_curve\n",
    "fpr,tpr,thrs = roc_curve(y_score=np.stack(all_logits)[sel,1], y_true=np.stack(all_gt_scores)[sel])\n",
    "plt.plot(fpr,tpr)\n",
    "plt.ylabel('TPR')\n",
    "plt.plot([0,1],[0,1],'k--')\n",
    "plt.xlabel('1-FPR')\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "auc = roc_auc_score(y_score=np.stack(all_logits)[sel,1], y_true=np.stack(all_gt_scores)[sel])\n",
    "plt.title('2-Level Hierarchical Classifier, Atypical/Typical decision, AUC=%.3f' % auc)\n",
    "plt.savefig('ROC_atypical_typical_2_level_run%d.svg' % run)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9adf0cc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEWCAYAAACT7WsrAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABSpElEQVR4nO3dd3gU1dfA8e8hQQEpUhUIPbQkhNB7EwREEBtFEAEpCqK8oiioP8CCioAgShWpSmiKoCCoFEERpYiUUA29EzqRkuS+f8xm3ZRNNiGbTbLn8zyB3dnZmTPbztx7Z86IMQallFIqMdk8HYBSSqmMS5OEUkoppzRJKKWUckqThFJKKac0SSillHJKk4RSSimnNEmkkojMEpH37uD5XUXkx7SMKbVE5LCItHDyWCMR2efOdbjw3KYicvxOY0hi+VNE5H8O9/uJyBkRuSYiBW3/l3XX+l2MMU0/L2n1vsZbZg8R+TUtl6k8L0MmCRG5W0S+EJEjInJVRLaLyEPJPCfVP0JpzdmXxTFGY8xXxpiW6R9dyhhjNhhjKrp7PSJSW0RWiMglEbkgIn+KSE93rxfAGPO8MeZdWxzZgY+BlsaY3MaYCNv/4e5Yt21nw4hI+3jTx9mm97DFGOfzYnvMPwXriTN/at9XERkhIrdtifOSiGwUkXopXU56sn0fd4pIpIicFpHJInKvp+Nylbtec1c/QxkySQC+wDGgCZAPeAtYKCKlPRlURiEiPimc39ddsaQF2wd+DfAL4A8UBPoBSe4YuMl9QA5g950uKAWv+37gmXjP6wj8c6cxuMkCY0xuoDDwK/CNiIiHY0qUiLwCjAIGY/2W1AVKAT+JyF2ejC2FYl/zQsBaYFF6rThDJgljzHVjzAhjzGFjTIwx5nvgEFAjpcsSkWwiMkRE/hGRCBFZKCIFbI/9ICID4s3/t4g8brtdSUR+su3Z7hORjmmxfbZlx2ltJLUu297mZNue9nWgmYg8LCJ/icgVETkmIiMc5i9t20voJSJHsX6AEZE+IrLH1joLE5HqDiGFiMgOEbksIgtEJIftOXG6ekSkhIh8IyLnbK/nZ7bp5URkjW3aeRH5KgV7a6OB2caYUcaY88ay1RiT6Ovt8H7GbsdjDo/5i8gvtu04LyILbNPFtnd+1vaa7RSRIIfX9z0RqQDEdsFcEpHY182+xyVWK3eMiBwVq0tqiojkdHytROR1ETkNzHRx+78DGopIftv91sAO4LTDdtk/LyKy3jb5b9veZSfb9D4ictD2GVomIsWcze/q+5oUY8xtYDZwP1Zij13WGBG5KCKHxKEHQER6Onz+wkXkOYfHConI9/JfS3KDiGSzPVZMRL62xXZIRF5y5UUVkbzA28CLxpiVxpjbxpjDWAm4NPC0bb4RYv0uzLHFtltEajosx6X1i0gdsVoqPg7THhORHbbbtUVki+3zd0ZEPnZlOxwZY6KAr4DiIlLYttx8YvW8nBKRE7bPso/tMWffh0Q/Q4nJkEkiPhG5D6hA6vbuXgQexWqVFAMuAhNtj4UCTzmsJwBrL2O5iNwD/ATMA4oAnYFJtnnSlIvr6gKMBPJg7b1dx9r7vBd4GOgnIo/GW3QToDLQSkQ6ACNsz8kLPAJEOMzbEevHqQwQDPRIJE4f4HvgCNaXrDgwP/Zh4AOs17gyUMK2vuS2PRdQD1ic3LwO/gEaYe0Zvg18KSJFbY+9C/wI5Af8gE9t01sCjbE+R/mwttdx+zHG7AcCbXfvNcY8kMi6P7QtIwSr1VMcGObw+P1AAazPUV8Xt+cGsBTrfQfrPZrjbGZjTGPbzaq2rrAFIvIA1uvfESiK9R7Ndza/4/KSeV+dEpG7sT4nx4wx522T62Al2kLAR8AXIvZWxlmgLdbnrycwTv7bUXkFOI7VOrkPeAMwtkTxHfC3La7mwP+JSKvk4gPqY7UKv3GcaIy5BqwAHnSY/Ihtm+8FlgGxOz8ur98Y8wfW99Lxc9MF63sN8AnwiTEmL1AOWOjCNsQhVuvnGazP7kXb5FlAFNbnsRrWZ7237bFEvw/JfSbib1iG/gOyAz8DU5OZ7zDQIpHpe4DmDveLArexurTyYL2ppWyPjQRm2G53AjbEW9ZUYLjt9izgPSex9LC9aZfi/cXExmib59cUrGtOMts/Hhhnu10aMEBZh8dXAQOTeO2edrj/ETDFdrspcNx2ux5wDvB14X17FPjLhfenuC3WSkksyx6Dk8e3A+1tt+cA0wC/ePM8gNWtUxfIFu8x+3vp8Nr5OjxusL6AYvu8lHN4rB5wyCHOW0COFHy+ZwHvAQ2B37F+pM4AObF2BnrE/7w4xuRw/wvgI4f7ubE+56WdzJ/a93WEbRsvYf3orwFqOMR40GHeXLb13u9kWd/GfiaBd7ASpX+8eeoAR+NNGwrMdCHWp4HTTh77EPjJYZt+dngsAPg3Neu3vZexvyHxf1/WY+3UFHL185HIax6NlSCa2h67D7gJ5HSY/ylgbVLfh8Q+E87+MnRLwpbF52K9QAMcpv9gayJdE5GuySymFLDE1oy9hJU0ooH7jDFXgeX8twf3FFZTLvZ5dWKfZ3tuV6w9RVdsMsbc6/gHHE0ixuTWdczxCbam7VpbE/gy8DzW3htOnlOCpPu4TzvcjsT6kYmvBHDEWE3eOETkPhGZb2vuXgG+TCSexFzESp5Fk5vRYV3PiHUwQ+xrFeSwrtewfsz/tHUbPAtgjFmDtXc4ETgrItNs3REpURjrh2+rw7pX2qbHOmeMuZHC5WKM+dW2nDeB740x/6ZwEcWwWgKxy7uG9WNS3IXnOn1fnVho+0wXMcY8YIzZ6vCY/XNkjIm03cwNICIPicgmW3fSJaAN/71vo4GDwI+2rqghtumlgGLxvhtvYP04Juc8UEgSHxsqans8QdxYn/8ctueldP3zgMdtrazHgW3GmNj3pRdWK3SviGwWkbYubEOshbbfkPuAXfzX9V4Ka0f6lEN8U7F6JMDJ9yElMuyApq2J+gXWi9LGWP2fABhjUjKgeQx41hjzm5PHQ4Hhtj66HFiDQrHP+8UY86CT56UlV9YVv1zvPKwfvYeMMTdEZDwJf5Qdn3MMq4l7p3GWFBHfRH5Q3retr4ox5oKt68uVfu1IEfkdeIL/XnunRKQU8DlWs/93Y0y0iGzH+iJgjDkN9LHN2xD4WUTWG2MOGmMmABNEpAhWU38w8L9EVuPMeeBfINAYc8LZJqVgefF9idV11SwVzz2J9YMB2LswCwLO4nSU1PuaJmw/ml9jdZUsNcbcFpFv+e99u4rV5fSKWGNFa0Rksy22Q8aY8qlY7e9Ye9mP49C1IyK5sQ6KeMOFZaRo/caYMBE5Ylu+Y1cTxpgDwFO2nd/HgcUiUtAYc93F7cEYc15E+gJbRGSeLb6bWK2TBO9dUt8HV9eZkVsSk7H6ttulYK8qu4jkcPjzBaYAI20/LohIYYl7uOEKrC/XO1hHEMTYpn8PVBCRbiKS3fZXS0Qqp8nWxZWadeUBLtgSRG2sD2RSpgOvikgNsfjHviYp8CdwCvhQRO6xvcYNHOK5BlwWkeJYP8Cueg3oISKDRaQggIhUFZHE+sXvwfohPmebrydWSwLb/Q4i4me7e9E2b4zt9awj1iGu17HGAWJIAdtn43OsvvQitvUVT65/XKyB76YurGICVj/5+uRmxOqScjx3IxToKSIhth/k94E/jDVQm9j8jpJ6X9PKXcDdWO9blFgD2o6H9La1fSYFuIzV2o+xxXZVrIMBcoqIj4gEiUgt2/OaikiiidkYcxmre+dTEWlt+16VxkoYx7F6KZKT5PqdmAcMxBoDsx+FJCJPi0hh2+fokm1yij6Dtu3ah9V9/Jox5hTWmMNYEckr1oE65USkiW2diX4fbPeT+kzYZcgkYfvxeg5rcPC0uN61tAJrTy/2bwTWYNEyrGbsVWATVj8jAMaYm1gDWy2Im/WvYn2IO2PtpZ3GOpTu7jvfwrhSua7+wDu2bRpGMoNgxphFWGMu84CrWP3BBVIYZzTQDqt//ijWFy32qIi3gepYX/DlxBssTGa5G7HGDB4AwkXkAlY/6opE5g0DxmLtJZ4BqgCOrcRawB8icg3rfR9orHMc8mL9wF/E6paJwOriSKnXsbpFNonVrfYz4PR8AxEpgfV670xuwcaYC8aY1cbWYZyMEcBsWxdDR2PMz1itoq+xfvDL8V83aoL54603qfc1Tdg+4y9hfU4vYu3ULHOYpTzWa3kN672dZIxZa4utLdZvwSGs1tx0rIMPwOoq25jEej/CajGMAa4Af2DtfTe3ffeTizu59ScmFOugkTXmvwF9sA4M2W37bH4CdI7dAbb9vjVKLh4Ho4G+tp2VZ7CScBjWa7uY/7pvnX0fIInPhCNx7fOolEoNEXkaq3tqqKdjyYpEZDqwyBizytOxZFWaJJRSSjmVIbublFJKZQyaJJRSSjmlSUIppZRTGfY8CWcKFSpkSpcu7ekwlFIqU9m6det5Y0zh5OeMK9MlidKlS7NlyxZPh6GUUpmK7SS/FNPuJqWUUk5pklBKKeWUJgmllFJOaZJQSinllCYJpZRSTrktSYjIDLEuFbnLyeMiIhPEutziDol7KU2llFIZgDtbErOwqh468xBW5cfyWJd5nOzGWJRSSqWC286TMMast9Vud6Y91iU5DVbZ5XtFpKitPrpKxrw/jrJ0uyvXk8kcauxYR5V9f3g6DK92b3QE+WIuAXAxm+GyT4ovdaDc5EZ0DJdvRXFPqfx0nLc5XdftyZPpihP38prHbdMSJAnblZj6ApQsWTJdgsvolm4/QdipKwQUTekVODPmD3Lp4/sAOOzn9NIMys3yxVwih7nBKZ+7OZU9GoBcMeLhqNTfF64zYe8p7vHNxnsl70339WeKM66NMdOwLkJDzZo1vbK2+cUFC7ny/ff2+z1OXQFIVZKI3GztieSqldTFteI69+85Iv6NSPG6XHW0XB721CjE3/Xzu20dKhmn7wbuZovcBHwZVm8YbSp08HRUXuvSpUsMHjyY6aun4+/vz/Tp02nSpEm6x+HJJHEC66pSsfxw7Xq8mU78H/jUSM0Pe3z2H3r7D7KPy8/dcuY4ADXvq5nq9Xudq6fh+jlPR+G6W9fhrnuoeV9N2pRtQwdNEB4THR1N/fr12bdvH6+99hojRowgZ86cHonFk0liGTDAdh3jOsDlzDoekVwSSIsf+Fy1apG3bVvyd7KuMvja1N8BeLLZcVaEJ7jKZ6Lu5IdefzhSYebDcPos3F/F05G4JjsQ9CTU7OnpSLxWREQEBQoUwMfHh5EjR1KiRAlq1vTsjpnbrkwnIqFAU6AQ1rWIh2N9DDHGTLFd8PwzrCOgIoGexphkK/fVrFnTZLQCf0e6PUPk5s1JJgHHH/iUWrR/UYJEEGbrborMth9w/Yc/0R/6LTNh5+JUxaaScHqnlSB6Lvd0JCqDM8bw1VdfMXDgQD788EP69OmT5usQka3GmBRnHHce3fRUMo8b4AV3rT+9XFyw0J4gSs2dk6bLjk0OW85YSTGxRJAme/g7F//3g6bSzv1VoMqTno5CZXDHjh3j+eefZ8WKFdStW5cGDRp4OqQ4MsXAdUYU28UU25V0oFZRRqxM22a6Y3KInwg62bqbZraulzYr0z1epdJdaGgozz33HNHR0YwfP54BAwbg4+P6WGF60CSRSrEJ4npQaX4NEL7ItwLOpO3AbopbCantNtJWhFIekT9/furUqcO0adMoU6aMp8NJlCaJVHDsYvqoqw9bzmzJGAO7qe020m4RpdJFVFQU48aN49atW7z55pu0bt2aVq1aYQ3RZkxemyRSc8byRZ/1hOxcRc9V1kFYEwscZ9vpa+QyFYg80pfFR2Dx2t+TXU7zyBU0+HdtquKO9eqtaHLd5QMz8/03UQdKlcqw/v77b3r16sXWrVvp2LEjxhhEJEMnCPDiJJHUGcsXfdZz2efPBNPr/b2XnqusUgUzWxXll5D85DD5yRddO0XrbvDvWkrfDudw9rKpCx7IdZcPhXLfHXeitgiUynBu3rzJe++9x4cffkiBAgVYtGgRTzzxRIZPDrG8NkmAdbbyguf+G/iNPZooLJGjiapuPEOrlVaCuP/tt/kolYezAra9/2oE6h6/UlnegQMHGDVqFF26dOHjjz+mYMGCng4pRbw6SThatH8R7/z+DpBwwPjigoWcXjgcsBJEnPMdUjNYrAPFSmVp165dY+nSpXTt2pWgoCD27t1L2bKp7znwJE0SxE0Qw+oNizP4fHHBQk4Pd5IgIHWDxdotpFSW9dNPP9G3b1+OHDlC9erVqVy5cqZNEKBJIskEAdjLbcRJEI6tBx0sVkoBFy9e5NVXX2XGjBlUqFCBX375hcqVK3s6rDvm9UkittxFYi2IK99/z429e8lVq1bcFoRj60FbBUp5vejoaBo0aMD+/fsZOnQow4YNI0eOHJ4OK014bZKIPYIp8sx+at5XM9EWxI29e8lRqRJ527ZNuABtPSjl9c6fP28vyPf+++9TsmRJqlfPWldiduflSzO0yz5/Epltv32Q2lHsyXI5KlWi1Nw55C933aroGft3eqeHolZKZQTGGObMmUOFChWYPn06AI8++miWSxDgxS0JgFwxFZjZemaC6bHjEPYWRPzBae1iUsprHTlyhOeee45Vq1ZRv359Gjdu7OmQ3Mork8S8P45y5d/b5M2Z3ek89nGILTPhyK9QqqF2Lynl5b788kv69euHMYZPP/2U/v37ky1b1u6Q8cokMWtnKL73HKJQThcOW409iklbDkp5vcKFC9OgQQOmTp1KqVKlPB1OuvDKJBFbcqNHyGOuPaFUQ71al1Je6Pbt24wdO5bbt2/zv//9j1atWtGyZctMU1IjLWTtdlIScsVUSLRia+ygNfBfV5NSyuv89ddf1KlTh6FDhxIWFkbsVTy9KUGAFycJZ+IMWmtXk1Je58aNG7zxxhvUqlWLkydP8vXXXxMaGup1ySGWJolE5KpVyzrsNXbAWrualPIaBw8eZMyYMTzzzDPs2bOHxx9/3NMheZQmCQdxupq0FaGU17h27Rpz584FICgoiH379jFjxgzy58/v4cg8T5OEjWMhP/v5EdqKUCrLW7VqFYGBgXTv3p09e/YAZNhLiXqCJgmbRAv5KaWyrIiICLp3707r1q3JlSsXGzZsyBIF+dKaVx4C60yCQn5KqSwptiDfwYMHefPNN3nrrbeyTEG+tKZJIr7YMuB6YSClspxz585RsGBBfHx8GDVqFKVKlSIkJMTTYWVo2t0Un2OC0EFrpbIEYwwzZ86kQoUKfP755wC0b99eE4QLNEkQ76gm+K8MuA5aK5XpHT58mFatWvHss89SpUoVmjVr5umQMhWvTxJxjmqqkl/PsFYqC5k7dy5BQUH8/vvvTJo0iXXr1lGhQgVPh5WpeH2SiHNUU+GD1kTtZlIqS7jvvvto3Lgxu3fvpl+/flm+Yqs76MD11dPkKpGD/JGzrbEIPTdCqUzr9u3bfPTRR0RHRzNs2DBatmxJy5YtPR1WpqZp9fo5uHXduq2D1UplWtu2baNWrVq89dZb7Nu3z16QT90ZbUkA3HWPXlBIqUzq33//5e2332bMmDEULlyYJUuW8Oijj3o6rCzDrS0JEWktIvtE5KCIDEnk8ZIislZE/hKRHSLSJrHlKKWUM+Hh4Xz88cf06NGDsLAwTRBpzG1JQkR8gInAQ0AA8JSIBMSb7S1goTGmGtAZmOSueGIt2r+IyGz73b0apZQbXblyhVmzZgEQGBjIgQMHmD59uhbkcwN3tiRqAweNMeHGmFvAfKB9vHkMkNd2Ox9w0o3xMO+Po4z5bb61suja7lyVUspNVqxYQVBQEL169bIX5POWS4l6gjuTRHHgmMP947ZpjkYAT4vIcWAF8GJiCxKRviKyRUS2nDt3LtUBLd1+gsibUeSKqUCPKk+lejlKqfR3/vx5unXrxsMPP0yePHn47bfftCBfOvD00U1PAbOMMX5AG2CuiCSIyRgzzRhT0xhTs3Dhwne0wlx3+xJQNC9d6pS8o+UopdJPbEG++fPnM2zYMLZt20bdunU9HZZXcOfRTSeAEg73/WzTHPUCWgMYY34XkRxAIeCsG+NSSmUSZ86coXDhwvj4+DBmzBhKlSpFcHCwp8PyKu5sSWwGyotIGRG5C2tgelm8eY4CzQFEpDKQA0h9f5JSKkswxvDFF19QsWJFpk2bBkC7du00QXiA25KEMSYKGACsAvZgHcW0W0TeEZFHbLO9AvQRkb+BUKCHSc8zYLbMhBuX0211SqnkhYeH06JFC3r37k1ISAgtWrTwdEheza0n0xljVmANSDtOG+ZwOwxo4M4YkhR7Het77mycQymVNmbPnk3//v3x8fFhypQp9OnTR+steZhXvfoXfdYnPEciRz7Ic79nAlJKxVGsWDEeeOABwsLCeO655zRBZABeVZbjss+fALTJdi/MfNgq6EdBj8aklDe7desWH374ITExMYwYMYIHH3yQBx980NNhKQdel6ZzxVSgw4n9/119TrualPKIzZs3U6NGDYYPH054eLgW5MugvC5J2MVefU67mpRKV5GRkbz66qvUrVuXixcvsmzZMubMmYOIeDo0lQjvTRIkctlSpZTbHTp0iE8//ZQ+ffqwe/du2rVr5+mQVBK8akwivtir0uVt29bDkSiVtV2+fJlvvvmGnj17EhgYyMGDBylRokTyT1Qe59UtCYBctWqRv1NHT4ehVJa1fPlyAgMD6d27N3v37gXQBJGJeH2SUEq5x7lz5+jatStt27Ylf/78/P7771SqVMnTYakU8uruJqWUe0RHR9OwYUMOHTrE22+/zZAhQ7jrrrs8HZZKBU0SSqk0c/r0aYoUKYKPjw9jx46ldOnSBAUFeTosdQe0u0kpdcdiYmKYOnUqFSpUYOrUqQC0bdtWE0QW4HVJIn90BBz51dNhKJVlHDx4kObNm/P8889Tq1YtWrVq5emQVBryuiSRN+aSdaPKkx6NQ6msYObMmVSpUoVt27bx+eef8/PPP1O2bFlPh6XSkNclCQBKNYSaPT0dhVKZXsmSJWnVqhVhYWH07t1bz5rOgnTgWinlsps3b/LBBx8QExPDO++8Q/PmzWnevLmnw1Ju5J0tCaVUiv3xxx/UqFGDt99+m6NHj2pBPi+hSUIplaTr168zaNAg6tWrx+XLl/n++++ZNWuWdi15CU0SSqkkHTlyhEmTJvH888+ze/duHn74YU+HpNKRjkkopRK4dOkSixcvpnfv3gQEBHDw4EH8/Pw8HZbyAK9pSSzavyjhpUuVUgksXbqUgIAAnn/+eXtBPk0Q3strksSK8BUANIi8i4vbr3Ck2zPcsH0BlFJw9uxZOnfuzKOPPkrhwoXZtGmTFuRT3tXdlCumAi0iz3Al7Bo3Lu0lR6VKei0JpbAK8jVo0ICjR4/y3nvv8dprr5E9e3ZPh6UyAK9KEo5yVKpEqblzPB2GUh518uRJ7r//fnx8fPjkk08oXbo0AQEBng5LZSBe090EVt2mYmH/EHnshqdDUcqjYmJimDx5MpUqVWLKlCkAtGnTRhOESsCrkkTemEtcOZLTuq3dTMpL7d+/n2bNmtG/f3/q1KnDQw895OmQVAbmVUkC4Hq2e/SSpcprffHFF1StWpUdO3YwY8YMfvzxR8qUKePpsFQG5rVjEkp5o9KlS/PQQw8xceJEihYt6ulwVCagSUKpLOzmzZu8++67ALz33ntakE+lmNd1NynlLTZu3EhISAgjR47k1KlTWpBPpYomCaWymGvXrjFw4EAaNmxIZGQkK1eu5IsvvtCCfCpV3JokRKS1iOwTkYMiMsTJPB1FJExEdovIPHfGo5Q3OHr0KFOnTuWFF15g165dejlRdUfcNiYhIj7AROBB4DiwWUSWGWPCHOYpDwwFGhhjLopIEXfFo1RWdvHiRRYtWkTfvn0JCAggPDycYsWKeToslQW4syVRGzhojAk3xtwC5gPt483TB5hojLkIYIw568Z4lMqSlixZQkBAAP3792ffvn0AmiBUmnFnkigOHHO4f9w2zVEFoIKI/CYim0SkdWILEpG+IrJFRLacO3fOTeEqlbmcPn2aDh068Pjjj3P//ffz559/UrFiRU+HpbIYTx8C6wuUB5oCfsB6EalijLnkOJMxZhowDaBmzZp6iIbyetHR0TRq1Ihjx47x/vvv8+qrr2pBPuUW7kwSJ4ASDvf9bNMcHQf+MMbcBg6JyH6spLHZjXEplWkdP36cYsWK4ePjw4QJEyhTpoyW81Zu5c7ups1AeREpIyJ3AZ2BZfHm+RarFYGIFMLqfgp3Y0xKZUoxMTF8+umnVKpUicmTJwPw0EMPaYJQbue2JGGMiQIGAKuAPcBCY8xuEXlHRB6xzbYKiBCRMGAtMNgYE+GumJTKjPbu3Uvjxo156aWXaNiwIW21OKVKR24dkzDGrABWxJs2zOG2AQbZ/pRS8UyfPp0BAwaQK1cuZs+eTbdu3fSkOJWuPD1wrZRKQrly5WjXrh2fffYZ9913n6fDUV5Ik4RSGciNGzd45513AHj//fdp1qwZzZo183BUyptp7SalMojffvuNkJAQPvjgA86dO6cF+VSGoElCKQ+7evUqL774Io0aNeLmzZusWrWKzz//XMceVIagSUIpDzt+/DjTp0/nxRdfZOfOnbRs2dLTISllp2MSSnlAREQECxcupF+/flSuXJnw8HC9UpzKkFLckhCRbCLS1R3BKJXVGWNYvHgxAQEBvPTSS/aCfJogVEblNEmISF4RGSoin4lIS7G8iHVGdMf0C1GprOHUqVM88cQTdOjQgRIlSrBlyxYtyKcyvKS6m+YCF4Hfgd7AG4AAjxpjtrs/NKWyjtiCfCdOnOCjjz7i5ZdfxtdXe3tVxpfUp7SsMaYKgIhMB04BJY0xN9IlMqWygGPHjlG8eHF8fHyYOHEiZcqUoUKFCp4OSymXJTUmcTv2hjEmGjiuCUIp10RHRzNhwoQ4BflatWqlCUJlOkm1JKqKyBWsLiaAnA73jTEmr9ujUyoT2rNnD7169eL333/noYceol27dp4OSalUc5okjDE+6RmIUlnBtGnTePHFF8mTJw9z586la9euelKcytScJgkRyQE8D/gDO4AZtvLfSiknypcvz2OPPcaECRMoUqSIp8NR6o4l1d00G2tcYgPQBggEBqZHUEplFv/++y8jRoxARPjwww+1IJ/KcpIauA4wxjxtjJkKPAk0SqeYlMoU1q9fT9WqVfnoo4+4fPmyFuRTWZKrRzdpN5NSNleuXKF///40adKE6OhoVq9ezeTJk3XsQWVJSXU3hdiOZgLriCY9ukkp4OTJk8yaNYtBgwbxzjvvcM8993g6JKXcJqkk8bcxplq6RaJUBnb+/HkWLlxI//79qVSpEocOHdIrxSmvkFR3k3awKq9njGHBggUEBATwf//3f+zfvx9AE4TyGkm1JIqIyCBnDxpjPnZDPEplGCdPnqRfv34sW7aMmjVrsnr1aj1jWnmdpJKED5Cb/864VsprREdH07hxY06cOMGYMWMYOHCgFuRTXimpT/0pY8w76RaJUhnAkSNH8PPzw8fHh0mTJlG2bFn8/f09HZZSHpPUmIS2IJTXiI6O5uOPP6Zy5cr2gnwtW7bUBKG8XlItiebpFoVSHrRr1y569erFn3/+Sdu2bXn00Uc9HZJSGYbTloQx5kJ6BqKUJ0yZMoXq1asTHh7OvHnzWLZsGX5+fp4OS6kMI8XXuFYqK4gtoVG5cmU6dOhAWFgYTz31lJ41rVQ8XnW4Ro0d0ciZKCjp6UiUp0RGRjJs2DB8fHwYNWoUTZo0oUmTJp4OS6kMy6taElX2RgOQt21bD0eiPGHdunUEBwczduxYrl27pgX5lHKBVyUJAHOfL/k7dfR0GCodXb58meeee85ewnvNmjVMnDhRu5aUcoHXJQnlfU6dOsWXX37Jq6++yo4dO/R6D0qlgFuThIi0FpF9InJQRIYkMd8TImJEpKY741He49y5c3z66acAVKpUicOHDzN69Ghy5crl4ciUylzcliRExAeYCDwEBABPiUhAIvPlwbri3R/uigWAq6fxMTFuXYXyPGMM8+bNo3Llyrzyyiv2gnyFCxf2cGRKZU7ubEnUBg4aY8KNMbeA+UD7ROZ7FxgF3HBjLHD9HACXs93r1tUozzl27Bjt2rWja9eu+Pv789dff2lBPqXukDuTRHHgmMP947ZpdiJSHShhjFme1IJEpK+IbBGRLefOnUt1QNGSjUs+BVP9fJVxRUVF0bRpU9auXcu4ceP47bffCAwM9HRYSmV6HjtPQkSyAR8DPZKb1xgzDZgGULNmTT1uUdkdPnyYEiVK4Ovry9SpUylbtixly5b1dFhKZRnubEmcAEo43PezTYuVBwgC1onIYaAusEwHr5UroqKiGDNmDJUrV2bSpEkAtGjRQhOEUmnMnS2JzUB5ESmDlRw6A11iHzTGXAYKxd4XkXXAq8aYLW6MSWUBO3bsoFevXmzZsoX27dvzxBNPeDokpbIst7UkjDFRwABgFbAHWGiM2S0i74jII+5arzNVt9+mzHHtqcrsJk2aRI0aNThy5AgLFixgyZIlFCtWzNNhKZVluXVMwhizAlgRb9owJ/M2dWcslXdHAbCzYh0ecueKlFsYYxARgoKC6Ny5M+PGjaNQoULJP1EpdUe8qsDfIT9ha3BTT4ehUuD69eu89dZb+Pr6Mnr0aBo3bkzjxo09HZZSXkPLcqgMa/Xq1VSpUoXx48dz8+ZNLcinlAdoklAZzqVLl+jduzctWrTA19eX9evXM2HCBC3Ip5QHaJJQGc6ZM2eYP38+r7/+On///TeNGjXydEhKeS2vGpNQGVdsYhg4cCAVK1bk8OHDOjCtVAagLQnlUcYYvvzySwICAnjttdc4cOAAgCYIpTIITRLKY44ePcrDDz9Mt27dqFixItu3b6d8+fKeDksp5UC7m5RHxBbkO3v2LBMmTKB///74+Ph4OiylVDyaJFS6Cg8Pp1SpUvj6+vL5559Trlw5Spcu7emwlFJOaHeTShdRUVGMGjWKgIAAJk6cCEDz5s01QSiVwWlLQrnd9u3b6dWrF9u2beOxxx6jQ4cOng5JKeUibUkot/rss8+oVasWJ06cYPHixXzzzTcULVrU02EppVykSUK5RWwJjeDgYLp27UpYWJiW9FYqE9LuJpWmrl27xptvvkn27NkZM2aMFuRTKpPTloRKMz/++CNBQUF8+umn3L59WwvyKZUFaEtC3bGLFy8yaNAgZs2aRcWKFVm/fj0NGzZM9nm3b9/m+PHj3LhxIx2iVMo75MiRAz8/P7Jnz54my9Mkoe7Y2bNnWbx4MUOHDmXYsGHkyJHDpecdP36cPHnyULp0aa3wqlQaMMYQERHB8ePHKVOmTJosU7ubVKqcPn2acePGAdgL8r3//vsuJwiAGzduULBgQU0QSqUREaFgwYJp2jrXJKFSxBjD7NmzCQgIYOjQofaCfAULFkzV8jRBKJW20vo7pUlCuezw4cO0bt2aHj16EBAQoAX5lPICmiSUS6KiomjWrBkbN25k4sSJrF+/nkqVKnk6rDvm4+NDSEgIQUFBtGvXjkuXLgFWQgwKCgJg3bp1iAjfffed/Xlt27Zl3bp1gPXavPHGG5QvX56QkBBCQkIYOXJkouubMWMGVapUITg4mKCgIJYuXQpAjx49WLx4cZx5c+fOHef++PHjyZEjB5cvX7ZPW7duHfny5SMkJITKlSvz9ttvJ5ge+/fzzz8n+3qcP3+e7NmzM2XKlCRjmTVrFgMGDLDfnzNnDkFBQVSpUoVq1aoxZsyYZNcV64MPPsDf35+KFSuyatWqROdZs2YN1atXJygoiO7duxMVFQXA5cuXadeuHVWrViUwMJCZM2fan/P6668TFBREUFAQCxYscDkeFZcmCZWkgwcPEh0dja+vLzNmzGDXrl3079+fbNmyxkcnZ86cbN++nV27dlGgQAF7Xan4/Pz8nP7wv/XWW5w8eZKdO3eyfft2NmzYwO3btxPMd/z4cUaOHMmvv/7Kjh072LRpE8HBwS7HGhoaSq1atfjmm2/iTG/UqBHbt29ny5YtfPnll2zbti3O9Ni/Fi1aJLuORYsWUbduXUJDQ12O64cffmD8+PH8+OOP7Ny5k02bNpEvXz6XnhsWFsb8+fPZvXs3K1eupH///kRHR8eZJyYmhu7duzN//nx27dpFqVKlmD17NgATJ04kICCAv//+m3Xr1vHKK69w69Ytli9fzrZt29i+fTt//PEHY8aM4cqVKy5vk/qP1xzdZOz/KFfcvn2b0aNH8/bbbzN69GheeuklmjVr5rb1vf3dbsJOpu2XOKBYXoa3C3R5/nr16rFjx45EH6tatSq3b9/mp59+4sEHH7RPj4yM5PPPP+fw4cP2Qfs8efIwYsSIBMs4e/YsefLkse+V586dO8EeujP//PMP165dY9KkSYwcOZKePXsmmOeee+6hRo0aHDx4kCJFiri03PhCQ0MZO3YsXbp04fjx4/j5+SX7nA8++IAxY8ZQrFgxAO6++2769Onj0vqWLl1K586dufvuuylTpgz+/v78+eef1KtXzz5PREQEd911FxUqVADgwQcf5IMPPqBXr16ICFevXsUYw7Vr1yhQoAC+vr6EhYXRuHFjfH198fX1JTg4mJUrV9KxY8dUvCreLWvsDrogxnZiV/uQ4h6OJOPbtm0btWvX5s0336R9+/Z06tTJ0yG5XXR0NKtXr+aRRx5xOs+bb77Je++9F2fawYMHKVmyJHny5El2HVWrVuW+++6jTJky9OzZM073FcDgwYPjdA85mj9/Pp07d6ZRo0bs27ePM2fOJFh+REQEmzZtIjDQSowbNmyIs7x//vknyfiOHTvGqVOnqF27Nh07dnS5i2bXrl3UqFEj0cdGjx4dJ4bYv5deegmAEydOUKJECfv8fn5+nDhxIs4yChUqRFRUFFu2bAFg8eLFHDt2DIABAwawZ88eihUrRpUqVfjkk0/Ili0bVatWZeXKlURGRnL+/HnWrl1rf45KGa9pSQAg0KVOSU9HkaFNmDCBQYMGUbhwYb755hsee+yxdFlvSvb409K///5LSEgIJ06coHLlynFaCfHFlhf59ddfnc4zc+ZMPvnkEyIiIti4cWOcH0AfHx9WrlzJ5s2bWb16NS+//DJbt261tzpGjx7Nk08+aZ/fsZURGhrKkiVLyJYtG0888QSLFi2yjwls2LCBatWqkS1bNoYMGUJgYCDr1q2jUaNGfP/99y6/FgsWLLDvaXfu3Jlnn32WV155xen8rhxFM3jwYAYPHuxyDM7WM3/+fF5++WVu3rxJy5Yt7ReoWrVqFSEhIaxZs4Z//vmHBx98kEaNGtGyZUs2b95M/fr1KVy4MPXq1dOLWqWS17QkVNJiS2hUq1aNZ555hrCwsHRLEJ4UOyZx5MgRjDFOxyRixW9N+Pv7c/ToUa5evQpAz5492b59O/ny5UvQtw7WD17t2rUZOnQo8+fP5+uvv042xp07d3LgwAEefPBBSpcuzfz58+OMGTRq1Ii//vqLrVu38vzzz7u66QmEhoYya9YsSpcuzSOPPMKOHTvshzjnzJmTW7du2ee9cOGC/TrkgYGBbN26NdFlJteSKF68eJw9/OPHj1O8eMLWfr169diwYQN//vknjRs3tnc9zZw5k8cffxwRwd/fnzJlyrB3717Aeq+2b9/OTz/9hDHG/hyVMpokvNzVq1cZMGAAr776KmD94MyYMYP8+fN7OLL0lStXLiZMmMDYsWPtR84kpmXLlly8eNE+dpErVy569erFgAED7CcwRUdHx/lBjXXy5En7oDJY19koVapUsrGFhoYyYsQIDh8+zOHDhzl58iQnT57kyJEjKd1MwLrYU/wunf3793Pt2jVOnDhhX8/QoUPtyahJkyZ8+eWXgNX6WrhwoX2MaujQoQwePJjTp08DcOvWLaZPnw5YLQnHwfPYvwkTJgDwyCOPMH/+fG7evMmhQ4c4cOAAtWvXThDz2bNnAbh58yajRo2yJ8OSJUuyevVqAM6cOcO+ffsoW7Ys0dHRREREALBjxw527NhBy5YtU/V6eTtNEl5s5cqVBAUFMWnSJIwxXl+Qr1q1agQHByd7ZM+bb74ZZ+935MiRFC1alKCgIKpVq0ajRo3o3r27fSA31u3bt3n11VepVKkSISEhLFiwgE8++STZuObPn5+gVffYY48xf/78JJ8Xf0xi8eLFxMTEcPDgQQoUKBBn3tDQ0ATreOKJJ+yvxSeffMI333xDSEgIdevWpUOHDvbutzZt2jBgwABatGhBYGAg1atXd/lIosDAQDp27EhAQACtW7dm4sSJ9m6hNm3acPLkScBqkVSuXJng4GDatWvHAw88AMD//vc/Nm7cSJUqVWjevDmjRo2iUKFC3L59m0aNGhEQEEDfvn358ssv8fX1rt71tCKZ7YehZs2aJnYAKyVWtAgAoM3PYWkdUqYTERHBoEGDmDNnDpUrV+aLL76IczRJetmzZw+VK1dO9/V6s127djFjxgw+/vhjT4ei3Cix75aIbDXG1EzpsrQl4YUiIiJYsmQJ//vf//jrr788kiCUZwQFBWmCUCni1iQhIq1FZJ+IHBSRIYk8PkhEwkRkh4isFpHkO2hVqpw6dYoxY8bYB/COHDnCO++8w9133+3p0JRSGZjbkoSI+AATgYeAAOApEQmIN9tfQE1jTDCwGPjIXfF4K2MMM2bMoHLlyvzvf//j4MGDAF43MK2USh13tiRqAweNMeHGmFvAfKC94wzGmLXGmEjb3U1A8qd3KpcdOnSIli1b0qtXL6pWrcrff/+tBfmUUinizuH+4oDjKY7HgTpJzN8L+CGxB0SkL9AXrEPeVPKioqJ44IEHiIiIYPLkyfTt2zfL1FtSSqWfDHFMmIg8DdQEmiT2uDFmGjANrKOb0jG0TOfAgQOULVsWX19fZs6cSbly5eKc9auUUinhzl3LE4Djr5OfbVocItICeBN4xBhz043xZGm3b9/mvffeIygoiM8++wyApk2baoJwwbfffouI2M/UBatUuIjw6aef2qcNGDCAWbNm2e9HRUVRuHBhhgyJe0xG06ZNqVixIlWrVqVBgwbs27fPPt3x8G3HcuSx/u///o/ixYsTExNjn3bmzBnatm1L1apVCQgIoE2bNvbn58yZM865EHPmzLE/b+fOnfbpBQoUoEyZMoSEhNCiRYskn1u6dGnOnz8fJ65bt27xf//3f/j7+1O+fHnat2/P8ePH7Y87llzv0KEDkZFWL3JyJcYBQkJC6Ny5c5xpmzZtok6dOvYS6LGlS2bNmkXhwoXjxB0Wlvxh7du3b0dEWLlyZZKv/4gRI+KUOR8zZoz9vJZatWrFeX2TYozhpZdewt/fn+Dg4DgnUTpasGABwcHBBAYG8vrrr9unv/zyy/btq1ChAvfee6/9sdatW3PvvffStm1bl2K5Y7EnUaX1H1YrJRwoA9wF/A0ExpunGvAPUN7V5daoUcOkxvLmlc3y5pVT9dyMbvPmzSY4ONgApnPnzubMmTOeDsklYWFhng7BGGNMx44dTcOGDc2wYcPs0w4dOmSKFCliypUrZ27evGmMMeaFF14wM2fOtM+zYsUKU79+fVO2bFkTExNjn96kSROzefNmY4wxU6dONe3atUswPXYdgYGB9vvR0dGmZMmSpk6dOmbNmjX26X379jXjx4+33//7778TfX5SunfvbhYtWuR03Y5KlSplzp07F2faK6+8Yp599lkTFRVljDFmxowZplatWvbtvueee+zzdunSxYwdOzbBdGOMmTlzpnnhhRfs98PCwkxQUJApVqyYuXbtmn16hQoVzPbt240xxkRFRZndu3cn+nxXvfbaa6Zhw4bmmWeesU9L7DUYPny4GT16tDHGmMmTJ5uWLVuay5cvG2OMuXz5spk1a5ZL61u+fLlp3bq1iYmJMb///rupXbt2gnnOnz9vSpQoYc6ePWuMMeaZZ54xP//8c4L5JkyYYHr27Gm///PPP5tly5aZhx9+2On6E/tuAVtMKn7L3dbdZIyJEpEBwCrAB5hhjNktIu/Ygl0GjAZyA4tsxcKOGmOcl+FUCXzyyScMGjSI+++/n6VLlyZZxTRD+2EInN6Ztsu8vwo89GGSs1y7do1ff/2VtWvX0q5dO/tFewAKFy5MgwYNmD17dqKlr0NDQxk4cCCTJ0/m999/p379+gnmady4MePHj3cp3HXr1hEYGEinTp0IDQ21l704depUnJISKbkGRVqIjIxk5syZHDp0yH42dM+ePZkxYwZr1qyhefPmceZv1KiR05Lr8YWGhtKtWzf27NnD0qVL6dKlC2CV4ShatChgtVICAuIfGOk6YwyLFi3ip59+olGjRty4ccOla7G///77rFu3jrx58wKQN29eunfv7tI6ly5dyjPPPIOIULduXS5dusSpU6fs2wQQHh5O+fLlKVy4MAAtWrTg66+/TvB6hoaGxvlcNm/e3H7Bq/Tg1pFMY8wKY0wFY0w5Y8xI27RhtgSBMaaFMeY+Y0yI7S+T/sKlP2M7U75mzZr06tWL3bt3Z94E4UFLly6ldevWVKhQgYIFCyYoVPf6668zZsyYBMX6bty4wc8//0y7du146qmnnJby+O6776hSpYr9fteuXe3dCLHdRrFCQ0N56qmneOyxx1i+fLn9wkUvvPACvXr1olmzZowcOdJeqgKs60w4dr1s2LDB5W139bmx5dBjfyxj1axZk927d8eZFhUVxQ8//GDf5tgqu7F/w4YNizP/ggUL6Ny5c4LX8OWXX6ZixYo89thjTJ061V4XK/Y5jsv8999/k9zOjRs3UqZMGcqVK0fTpk1Zvnx5sq/NlStXuHr1KmXLlk30ccfuIMe/Dz+0dkpcKYHu7+/Pvn37OHz4MFFRUXz77bcJypkfOXKEQ4cO2cuQeEKGGLhWrrty5Qqvv/46OXLkYNy4cTRo0IAGDRp4Oqw7l8wev7vEtgbAKo8dGhoa59oIZcuWpU6dOsybNy/O877//nuaNWtGzpw5eeKJJ3j33XcZP368fU+7a9eu5MyZk9KlS8cZ1/jqq6+oWdOqjHD48GF7v/KtW7dYsWIFH3/8MXny5KFOnTqsWrWKtm3b0qpVK8LDw1m5ciU//PAD1apVY9euXQCUK1eO7du3p2rb7+S58cUmA7BaEr169QL+q7Iba9asWfZxmS1btlCoUCFKlixJ8eLFefbZZ7lw4QIFChRg2LBhdO3alR9//JF58+YRGhpq33vu1KmTfdzNFaGhofYxj86dOzNnzhyeeOIJp6XOXSmBPm7cOJfX70z+/PmZPHkynTp1Ilu2bNSvXz/BNT/mz5/Pk08+6dEy55okMpEVK1bw3HPPcfLkSQYNGoQxxqUPtErchQsXWLNmDTt37kREiI6ORkQYPXp0nPneeOMNnnzySZo0+e/gu9DQUH799VdKly4NWKVO1qxZY78ehWMycMWqVau4dOmSfQ88MjKSnDlz2pNIgQIF6NKlC126dKFt27asX7/e6YV+lixZYu+emD59eoriSEy5cuXs5dAdL660detWe3zxk4ErQkND2bt3r/01vHLlCl9//bW9a69cuXL069ePPn36ULhwYXtV15SIjo7m66+/ZunSpYwcORJjDBEREVy9epWCBQty8eLFOPNfuHCBMmXKkDdvXnLnzk14eHiirYmXX36ZtWvXJpjeuXNnhgwZ4nIJ9Hbt2tGuXTsApk2bliAZzJ8/P9ny9e6mB85nAufPn+fpp5/m4YcfJl++fGzcuJHRo0drgrhDixcvplu3bhw5coTDhw9z7NgxypQpk6DbpVKlSgQEBNivJHflyhU2bNjA0aNH7WW1J06cmKLrQscXGhrK9OnT7cs7dOgQP/30E5GRkaxZs8Z+tNDVq1f5559/kjxf6LHHHrOX5L7TBAHWZVG7d+/OoEGD7N1uc+bMITIyMtXdIDExMSxcuJCdO3fat3np0qX213D58uX2LtUDBw7g4+MT5wifxFSqVCnBtNWrVxMcHMyxY8c4fPgwR44c4YknnmDJkiXkzp2bokWLsmbNGsBKECtXrqRhw4aAVQL9hRdesFe0vXbtmv3opnHjxiVaAj32SLdHHnmEOXPmYIyxX/PbcTwiVmwJ9IsXLzJp0iR69+5tf2zv3r1cvHjR47XVNElkAhcvXuS7775j+PDhbNu2jTp1kjonUbkqufLYjt588037IZ9LlizhgQceiFP3qn379nz33XfcvJnyo7gjIyNZuXIlDz/8sH3aPffcQ8OGDfnuu+/YunUrNWvWJDg4mHr16tG7d29q1aoFJBxXiL1OgyuSem5wcDB+fn74+fkxaNAgPvjgA3LkyEGFChUoX748ixYtYsmSJaneUdmwYQPFixePU069cePGhIWFcerUKebOnUvFihUJCQmhW7dufPXVV/a97PhjEhs3buT8+fOJlrpP7j2eM2cO7777LiEhITzwwAMMHz6ccuXKAdCvXz+aNWtGrVq1CAoKolGjRi6fkNqmTRvKli2Lv78/ffr0YdKkSfbHYrvlAAYOHEhAQAANGjRgyJAhcS6MFHvJ2vivcaNGjejQoQOrV6/Gz8+PVatWuRRTammp8AzqxIkTfPXVVwwePBgR4dKlS8nuSWU2WipcpZXvv/+e8PBw+xXvvF1algrXMYkMxhjD9OnTefXVV7l9+zaPP/44/v7+WS5BKJWW0u3EMi+k3U0ZyD///EPz5s3p27cv1atXZ8eOHfj7+3s6LKWUF9OWRAYRFRVF8+bNuXDhAlOnTqV3795akE8p5XGaJDxs3759lCtXDl9fX2bPnk25cuXw89OK6UqpjEF3VT3k1q1bvP3221SpUsV+HHSTJk00QSilMhRtSXjAn3/+Sa9evdi1axddunSha9eung5JKaUSpS2JdDZ+/Hjq1atnP/fhq6++olChQp4Oy2s5K3HtOL1du3ZcunTJ/pzdu3fzwAMPULFiRcqXL8+7775rP0Z/1qxZiAg///yzff7YUuSLFy8G/islHnuc/5NPPsnIkSPt92PX7XjuQmIlxB3LZgcEBPD555/bp8cvxx2/THlipbMBRo4cSWBgIMHBwYSEhPDHH384jdkVicUdvxw3xC1Pfvr0aTp37ky5cuWoUaMGbdq0Yf/+/S6t79ChQ9SpUwd/f386derErVu3Esxz69YtevbsSZUqVahatWqcYnmhoaFUqVKF4OBgWrduHadk+qeffkqlSpUIDAzktddecymeLCE1pWM9+ZdZS4XHllT+7bffzHPPPWcuXbrksVgyioxQKtyVEtfPPPOMee+994wxxkRGRpqyZcuaVatWGWOMuX79umndurX57LPPjDFWKesqVaqYXr162Z/fsWNHU7VqVXup7vglw5OKyRjnJcQdy2afOXPGFCpUyJw+fTrRctrx15lY6eyNGzeaunXrmhs3bhhjjDl37pw5ceKESzEnxlncjuW4Y8WWJ4+JiTF169Y1kydPtj+2fft2s379epfW2aFDBxMaGmqMMea5554zkyZNSjDPZ599Znr06GGMsV636tWrm+joaHP79m1TuHBhe5n0wYMHm+HDhxtjjFmzZo1p3ry5/bXJ6OX4M0WpcGW5fPkyr732Gjlz5mT8+PHUr18/0ZLS3m7Un6PYe2Fv8jOmQKUClXi99uvJz2jjrMR1vXr17NPnzZtHgwYN7KW7c+XKxWeffUbTpk154YUX7MvZsGEDt2/f5ubNmxw8eDDOWbYp5ayEuKMiRYpQrlw5jhw5kuzyjJPS2adOnaJQoUL2M8nvtIXrStzxrV27luzZs/P888/bp1WtWtWl9RljWLNmjb0YY/fu3RkxYgT9+vWLM19YWJi9nEiRIkW499572bJlC9WqVcMYw/Xr1ylYsCBXrlyxH4I+efJkhgwZYn9tihQp4lJMWYF2N7nRd999R0BAANOnT+fuu+9OtGyAyhjil7iOFR0dzerVq+1l2Hfv3p2gsF65cuW4du2avcaPiNCiRQtWrVrl9BofjiXDBw8enGRszkqIOwoPDyc8PNz+oxa/dIVjV5Oz0tktW7bk2LFjVKhQgf79+/PLL7+kOmZX445v165dTgsXXr16NdHy3LFXp4uIiODee+/F19fa902sPDdYSWfZsmVERUVx6NAhtm7dyrFjx8iePTuTJ0+mSpUqFCtWjLCwMHs12/3797Nhwwbq1KlDkyZN2Lx5c7LbklVoS8INzp07x8CBA+39m99++6291o5KXEr2+NOSsxLXsdNPnDhB5cqV7dVdXdW5c2cmTJjA5cuXGTt2LO+//36cx12tEptUCXGwksGvv/7K3XffzdSpUylQoACQsJx206ZN7bedlc7OnTs3W7duZcOGDaxdu5ZOnTrx4Ycf0qNHjxTFnFzcqS3RnSdPniQrzca/5Kozzz77LHv27KFmzZqUKlWK+vXr4+Pjw+3bt5k8eTJ//fUXZcuW5cUXX+SDDz7grbfeIioqigsXLrBp0yY2b95Mx44dCQ8P94oim5ok3ODy5cusWLGCt99+myFDhnDXXXd5OiTlhLMS17HTIyMjadWqFRMnTuSll14iICCA9evXx5k3PDyc3Llzx7koT+3atdm5cye5cuWKU7QtpZIrIZ7SayskVTo7T548+Pj40LRpU5o2bUqVKlWYPXu2PUmkVdwFCxbk1KlTcea/evUq9957L4GBgfYB/viuXr1Ko0aNEn1s3rx5VK5cmUuXLhEVFYWvr6/T8ty+vr5xrgdRv359KlSoYP8cxBb469ixo/0iQn5+fjz++OOICLVr1yZbtmycP3/eflW5rEy7m9LIsWPH+OCDDzDG4O/vz5EjRxg2bJgmiEwuV65cTJgwgbFjxxIVFUXXrl359ddf7Ucv/fvvv7z00kuJHu3y4YcfJmhBpFRSJcRTI6nS2fv27ePAgQP2ebdv306pUqWSXN6SJUsYOnRoiuJu3Lgxy5Yt4+rVqwB88803VK1aFR8fHx544AFu3rzJtGnT7MvasWMHGzZssLckEvsLCAhARGjWrJk9ycyePZv27dsniC0yMpLr168D8NNPP+Hr60tAQADFixcnLCyMc+fO2R+LLZL36KOP2q8fsX//fm7duuU1RyVqS+IOxcTEMG3aNF577TWio6Pp0KED/v7+5MuXz9OhqTRSrVo1goOD7ddjXrp0KS+++CIvvPAC0dHRdOvWLcEhpwAPPfSQ02XGXrkOrAFix0NmY8WWEJ8yZYp9mmMJ8dRwVjp78uTJBAYG8uKLL3Lp0iV8fX3x9/eP82OdWMz//PNPgsuaJhd3p06dGDBgAA0bNkREKFKkCNOnTwesLqclS5bwf//3f4waNYocOXJQunRpl68TPmrUKDp37sxbb71FtWrV7N2Hy5YtY8uWLbzzzjucPXuWVq1akS1bNooXL87cuXMBKFasGMOHD6dx48Zkz56dUqVKMWvWLMDqonr22WcJCgrirrvuYvbs2V7R1QRaKvyOHDhwgD59+vDLL7/QvHlzpk2b5vSauCohLRWe+T399NOMGzfOK7pdMhMtFZ4BREVF8eCDD3Lp0iW++OILevbs6TV7FkrF+vLLLz0dgnIzTRIptGfPHsqXL4+vry9z586lXLlyca6upZRSWYkOXLvo5s2bDB8+nODgYPvRJI0aNdIEoZTK0rQl4YJNmzbRq1cvwsLC6NatG926dfN0SEoplS60JZGMsWPHUr9+fa5evcqKFSuYM2cOBQsW9HRYSimVLjRJOBFbtbJevXo8//zz7Nq1K8lDGpVSKivSJBHPpUuX6NWrFwMHDgSsszEnTZqU4FhwlTWICE8//bT9flRUFIULF7af0exYjjv27++//7bfLlCgAGXKlCEkJIQWLVoAzstwOytLnjt37jjzJVbqOyQkxF5KI9amTZuoU6cOISEhVK5cmREjRjiNOSws+UO/E4v78OHDBAUFxZkvfqnvMWPGUKlSJUJCQqhVqxZz5sxJdl1gFeR76aWX8Pf3Jzg4mG3btiU634IFCwgODiYwMJDXX/+vfMvRo0dp1qyZ/TyWFStW2B/bsWMH9erVIzAwkCpVqnDjxg2XYlIJaZJw8O233xIQEMDs2bPJkyePFuTzAvfccw+7du3i33//BayzbOOXcujUqVOcs3urVq1qv/3II48wevRotm/fbj8hLjQ0lIYNGxIaGhpnObGlPnbt2sVdd90V52SzpOzZs4fo6Gg2bNhgP1MYrCqn06ZNsy+zY8eOTmMOCAhIdj3O4k7KlClT+Omnn/jzzz/Zvn07q1evdvl788MPP3DgwAEOHDjAtGnTElRrBYiIiGDw4MGsXr2a3bt3c/r0aVavXg3Ae++9R8eOHfnrr7+YP38+/fv3B6xE//TTTzNlyhR2797NunXryJ49u8vbpOLSgWvg7NmzDBgwgEWLFhESEsL3339P9erVPR2WVzn9/vvc3JO2pcLvrlyJ+994I9n52rRpw/Lly3nyySftlUs3bNiQqnU6K8Mdn7Oy5ImJPdN7z549LF26lC5dugDW57Zo0aKA1UpxJRHcadzxvf/++6xbt87e0s6bNy/du3d3aZ1Lly7lmWeeQUSoW7culy5d4tSpU/ZtAqsuVvny5e0n67Vo0YKvv/6a5s2bIyL2yruXL1+2H2n4448/EhwcbC8xrmOId0ZbEsCVK1f46aefGDlyJH/++acmCC/TuXNn5s+fz40bN9ixYwd16tSJ83j8stuxrY7EOCvD7Sh+WfLYirOxf8OGDUuw/s6dO/PUU0/F2ct/+eWXqVixIo899hhTp06N06WSkphdjTu+K1eucPXqVadVBl5++eVEy3rHFs07ceIEJUqUsM+fWGlvf39/9u3bx+HDh4mKiuLbb7/l2LFjgNXt9eWXX+Ln50ebNm349NNPAau2kojQqlUrqlevzkcffZTstijnvLYlcfToUebOncsbb7yBv78/R48eJU+ePJ4Oy2u5ssfvLsHBwRw+fJjQ0FDatGmT4PGUVFp1VoYbnJclj1+JdtasWfbrP2zZsoVChQpRsmRJihcvzrPPPsuFCxcoUKAAw4YNo2vXrvz444/MmzeP0NBQ+6U4U1od1lncqS3rDcSptJpa+fPnZ/LkyXTq1Ils2bJRv359/vnnH3vMPXr04JVXXuH333+nW7du7Nq1i6ioKH799Vc2b95Mrly5aN68OTVq1KB58+Z3HI83cmuSEJHWwCeADzDdGPNhvMfvBuYANYAIoJMx5rA7Y4qJiWHKlCm8/vrrxMTE0KlTJ/z9/TVBeLlHHnmEV199lXXr1hEREZGqZSRXhttZWfKkhIaGsnfvXkqXLg1Ye+9ff/01ffr0Aayy1v369aNPnz4ULlw4VbEnFXfBggW5ePFinPkvXLhAmTJlyJs3L7lz5yY8PDzR1sTLL79sr5zqqHPnzgwZMoTixYvbWwWA09Le7dq1o127dgBMmzYNHx8fAL744gv7IHu9evW4ceMG58+fx8/Pj8aNG9urtLZp04Zt27Zpkkglt3U3iYgPMBF4CAgAnhKR+J2mvYCLxhh/YBwwyl3xABy/ftN+mcl69eqxe/du+5W8lHd79tlnGT58eIIr06VEUmW4UyMmJoaFCxeyc+dOe8ntpUuX2rucli9fbh8kPnDgAD4+Ptx7771JLrNSpUopijt37twULVqUNWvWAFaCWLlyJQ0bNgRg6NChvPDCC/axgWvXrtmPbho3blyiZb2HDBkCWIl5zpw5GGPYtGkT+fLlizMeEevs2bMAXLx4kUmTJtG7d28ASpYsaR/E3rNnDzdu3KBw4cK0atWKnTt3EhkZSVRUFL/88ssdjdd4O3eOSdQGDhpjwo0xt4D5QPzi7u2B2bbbi4Hm4qYqedExhv9tP8bOnTuZOXMmq1atsu+dKeXn58dLL72U6GPx+/c3btyY6HzOynCn5GghRxs2bKB48eJxSr80btyYsLAwTp06xdy5c6lYsSIhISF069aNr776yr6XnVjM58+fT/TIo+TinjNnDu+++y4hISE88MADDB8+3H5hnn79+tGsWTNq1apFUFAQjRo1Ils2135W2rRpQ9myZfH396dPnz5MmjTJ/pjjNcEHDhxIQEAADRo0YMiQIfaLOI0dO5bPP/+cqlWr8tRTTzFr1ixEhPz58zNo0CBq1apFSEgI1atX5+GHH3YpJpWQ20qFi8iTQGtjTG/b/W5AHWPMAId5dtnmOW67/49tnvPxltUX6AtQsmTJGq5c7D2+hV1qsffcVfrMWZvo3opKf1oqPH19//33hIeHO02GKuvwulLhxphpwDSwrieRmmV0nOc9Fy5XKjGxJwgqlRLu7G46AZRwuO9nm5boPCLiC+TDGsBWSimVAbgzSWwGyotIGRG5C+gMLIs3zzIg9sybJ4E1Rk9z9ir6diuVttL6O+W2JGGMiQIGAKuAPcBCY8xuEXlHRB6xzfYFUFBEDgKDgCHuikdlPDly5CAiIkIThVJpJPbwZVfOlneV11zjWmU8t2/f5vjx41p8Tak0lCNHDvz8/BLUq8rSA9cqa8qePTtlypTxdBhKqSRo7SallFJOaZJQSinllCYJpZRSTmW6gWsROQek/JRrSyHgfLJzZS26zd5Bt9k73Mk2lzLGFE7pkzJdkrgTIrIlNaP7mZlus3fQbfYOnthm7W5SSinllCYJpZRSTnlbkpjm6QA8QLfZO+g2e4d032avGpNQSimVMt7WklBKKZUCmiSUUko5lSWThIi0FpF9InJQRBJUlhWRu0Vkge3xP0SktAfCTFMubPMgEQkTkR0islpESnkizrSU3DY7zPeEiBgRyfSHS7qyzSLS0fZe7xaReekdY1pz4bNdUkTWishfts93G0/EmVZEZIaInLVduTOxx0VEJthejx0iUt2tARljstQf4AP8A5QF7gL+BgLizdMfmGK73RlY4Om402GbmwG5bLf7ecM22+bLA6wHNgE1PR13OrzP5YG/gPy2+0U8HXc6bPM0oJ/tdgBw2NNx3+E2NwaqA7ucPN4G+AEQoC7whzvjyYotidrAQWNMuDHmFjAfaB9vnvbAbNvtxUBzEZF0jDGtJbvNxpi1xphI291NWFcKzMxceZ8B3gVGAVmhHrkr29wHmGiMuQhgjDmbzjGmNVe22QB5bbfzASfTMb40Z4xZD1xIYpb2wBxj2QTcKyJF3RVPVkwSxYFjDveP26YlOo+xLo50GSiYLtG5hyvb7KgX1p5IZpbsNtua4SWMMcvTMzA3cuV9rgBUEJHfRGSTiLROt+jcw5VtHgE8LSLHgRXAi+kTmsek9Pt+R/R6El5GRJ4GagJNPB2LO4lINuBjoIeHQ0lvvlhdTk2xWovrRaSKMeaSJ4Nys6eAWcaYsSJSD5grIkHGmBhPB5YVZMWWxAmghMN9P9u0ROcREV+sJmpEukTnHq5sMyLSAngTeMQYczOdYnOX5LY5DxAErBORw1h9t8sy+eC1K+/zcWCZMea2MeYQsB8raWRWrmxzL2AhgDHmdyAHViG8rMql73tayYpJYjNQXkTKiMhdWAPTy+LNswzobrv9JLDG2EaEMqlkt1lEqgFTsRJEZu+nhmS22Rhz2RhTyBhT2hhTGmsc5hFjTGa+9q0rn+1vsVoRiEghrO6n8HSMMa25ss1HgeYAIlIZK0mcS9co09cy4BnbUU51gcvGmFPuWlmW624yxkSJyABgFdaRETOMMbtF5B1gizFmGfAFVpP0INYAUWfPRXznXNzm0UBuYJFtjP6oMeYRjwV9h1zc5izFxW1eBbQUkTAgGhhsjMm0rWQXt/kV4HMReRlrELtHZt7pE5FQrERfyDbOMhzIDmCMmYI17tIGOAhEAj3dGk8mfi2VUkq5WVbsblJKKZVGNEkopZRySpOEUkoppzRJKKWUckqThFJKKac0SSiViOQqcdrmiRaR7Q5/pUWkqYhctt3fIyLDbfM6Tt8rImPSb2uUSj1NEkolbhaQXN2jf40xIQ5/h23TNxhjQrDKnzztUMo5dno1oK2INEj7sJVKW5oklEqEC5U4XVnGdWAr4B9v+r/AdtxYlE2ptKJJQqnUy+nQ1bQk/oMiUhCrZtTueNPzY9VTWp8+YSqVelmuLIdS6ehfW/dRfI1E5C8gBvjQVkaiqW3631gJYrwx5nS6RapUKmlLQikXiEgJh1bD88nMvsEYU80YU8NWa8dxelUgEOglIiFuC1ipNKItCaVcYIw5BoSk0bIOiciHwOtY10JQKsPSloRSibBV4vwdqCgix0WkVxqvYgrQWERKp/FylUpTWgVWKaWUU9qSUEop5ZQmCaWUUk5pklBKKeWUJgmllFJOaZJQSinllCYJpZRSTmmSUEop5dT/AzLPWhgGiTVEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sel = np.stack(all_gt_scores_l2)>=0\n",
    "from sklearn.metrics import roc_curve\n",
    "for i in range(4):\n",
    "    fpr,tpr,thrs = roc_curve(y_score=np.stack(all_logits_l2)[sel,i], y_true=np.stack(all_gt_scores_l2)[sel]==i)\n",
    "    auc = roc_auc_score(y_score=np.stack(all_logits_l2)[sel,i], y_true=np.stack(all_gt_scores_l2)[sel]==i)\n",
    "    plt.plot(fpr,tpr, label=f'{list(categories_l2.values())[i+1]}, AUC={auc:.3f}')\n",
    "\n",
    "plt.ylabel('TPR')\n",
    "plt.plot([0,1],[0,1],'k--')\n",
    "plt.xlabel('1-FPR')\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "plt.legend()\n",
    "plt.title(f'2-Level Hierarchical Classifier, Mitotic Phase, One vs. Rest')\n",
    "\n",
    "#, AUC={auc:.3f}'  )\n",
    "plt.savefig('ROC_mitotic_phase_2_level_run%d.svg' % run)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "9cbc6e2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix,ConfusionMatrixDisplay\n",
    "\n",
    "cm1 = confusion_matrix(y_true=all_gt_scores, y_pred=all_predictions)\n",
    "cm2 = confusion_matrix(y_true=all_gt_scores_l2, y_pred=all_predictions_l2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "eeae9d7f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7f10c61396a0>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWQAAAEwCAYAAACABAvBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAugUlEQVR4nO3dd7hU1dn38e/vFHrviKioBIPYEClqDLYoxgT10Wg0ioYnxNiiqZrkjd2YqrHmwRKxYVewIQRFxdhooqAIKkhHeodT7vePvQ6McMoAM2fvmXN/vPZ19l67zD0j1z1r1l57LZkZzjnn4lcQdwDOOecinpCdcy4hPCE751xCeEJ2zrmE8ITsnHMJ4QnZOecSwhOyc87tAElXSJom6SNJwyU1kNRF0ruSZkl6XFK9cGz9sD0r7N+r2mt7P+SdV0/1rQGN4w4jscpbNIo7hOST4o4g0TatW07JpnW79CGdcHRjW7a8LK1jJ07d9IqZnVjVfkmdgPFAdzPbIOkJ4CXgJOAZM3tM0r+AD8zsbkkXAQea2YWSzgJONbMzq7p+0Q68L7eNBjSmj46NO4zE2nB077hDSLyyev4jtTofjr51l6+xdHkp/x3VKa1jG+z2RZs0DisCGkoqARoBC4FjgLPD/mHANcDdwMCwDvAUcIckWRU1Yf/X4JzLawaUY2ktNV7LbD7wN+BLokS8CpgIrDSz0nDYPKDiG6ATMDecWxqOb13V9T0hO+fyXnma/wFtJE1IWYakXkdSS6JabxdgN6AxUGUTx47yJgvnXF4zjLL075UtNbNe1ew/DvjCzL4CkPQMcATQQlJRqAXvDswPx88HOgPzJBUBzYFlVV3ca8jOubyXqSYLoqaKvpIaSRJwLDAdeA04PRwzCBgR1keGbcL+V6tqPwavITvn8pwBZekl25qvZfaupKeASUApMBkYCrwIPCbphlB2XzjlPuAhSbOA5cBZ1V3fE7JzLq8ZUGLlmbue2dXA1dsUfw5s163IzDYCZ6R7bU/Izrm8l7l0nF2ekJ1zec2wjDVZZJsnZOdcfjMoy4187AnZOZffogdDcoMnZOdcXjNEieXGmCGekJ1zea8MT8jOORe7qB+yJ2TnnEuEcm+ycM65+HkN2TnnEsIQZTkybI8nZOdcXosenfaE7JxzCSDKPCE751z8ogdDPCE751wi+E0955xLADNvsnDOucQo9xqyc87FzxCbLTdSXW5E6ZxzOymXburlRpTOObcLykxpLTWR1E3SlJRltaTLJbWSNEbSzPC3ZThekm6TNEvSVEk9q7u+J2TnXF6reFIvnaXGa5nNMLODzexg4FBgPfAscCUw1sy6AmPDNsAAoGtYhgB3V3d9T8jOubxXbgVpLTvoWOAzM5sDDASGhfJhwClhfSDwoEXeAVpI6ljVBb0N2TmX18oRm60w3cPbSJqQsj3UzIZWcexZwPCw3t7MFob1RUD7sN4JmJtyzrxQtpBKeEJ2zuW9Hbipt9TMetV0kKR6wPeBq7bdZ2Ymaadm8fOEnCd69V/NhdcvoLDAeHl4K564o33NJ+WZdi3X8rtB42jVdANm8Pxb3+Sp13rQtNFGrhn8Kh1br2HhsqZcfe+xrN1QHzAuO+Nt+u4/l00lRfzpwW/z6dw2cb+NrGnXYi3/75zXaNl0PZgY8fY3efKNA7j4+29zxP5fUlJWwPylzbhpeH/WbqhPs0YbufGCMey3xxJefq8b/3j6yLjfwk4xIxsPhgwAJpnZ4rC9WFJHM1sYmiSWhPL5QOeU83YPZZVKbBuypHGSqv2mkvQtSdPC3c5Okp7K0Gv3l/RCJq5VGwoKjItvms8fzunCT/p34+iBK9mj68a4w6p1ZWUF3PV0X867/gwu/OtATj1qGnt2WME5J3zApBm7cfY1ZzJpxm786IQpAPTdfy67t1vF2df8gL8+ciS/OGt8vG8gy8rKxe0j+vKjm89kyK2ncNqR09ir/Qren7E75/75DAb95QzmftWcc4+bDMDm0kLueakXd47oF3Pku0qUp7nsgB+ytbkCYCQwKKwPAkaklJ8Xelv0BValNG1sJ7EJOU3nAH8Kdz3nm9npcQcUh26HrGfB7Hos+rI+pSUFjBvRgn4nrIo7rFq3bHWjLTXcDZvqMWdRS9q2WMeRB85h1DvfAGDUO9/gyIPmAHDkgXN45d2ugJg+uz1NGm2mdbP1cYWfdctWN+bTeW0BWL+pHnMWt6Bt83W8N6MzZeVRKpg2uz3tmq8DYOPmYqZ+0ZHNpWm3vyaSEdWQ01nSIakxcDzwTErxzcDxkmYCx4VtgJeAz4FZwD3ARdVdO2tNFpL2Al4wsx5h+1dAE6A/8C5wNNACGGxmb0pqCPwbOAj4BGiYcq3vANcC9YHPgAuIGtR/AJwgaQDw+4rXk3Q+UftOI2Af4Fkz+01V1zKztZJOBG4l6saSU1Wl1h1K+GpBvS3bSxcWs1/P/E0s6ejQag1dOy9l+ux2tGy6gWWrGwGwbHVDWjbdAECbFutYsqLJlnO+WtGYNi3WbTk2n3VotYauuy9j2px2Xyv/bp9PGDt5n5iiyp5MDlBvZuuA1tuULSPqdbHtsQZcnO6146ohF5lZb+By4OpQ9jNgvZl9M5QdCiCpDfAH4Dgz6wlMAH5hZvcS/Rz4tZmdU8lrHAycCRwAnCmpc1XXktSA6Nvre+F1O2T+Lbva0rB+CdcP+Q+3P9WP9RvrbbM3N8Y0yKaG9Uq48YLR3PZsP9Zv2vr5nHf8JMrKCxg9sWuM0WWeIUqsMK0lbnHd1Kuo6k8E9grrRwG3AZjZVElTQ3lfoDvwliSAesDbabzGWDNbBSBpOrAnUY28smvtB3xhZjPD8Q8TdeLejqQhFfsakIya1LJFxbTdbfOW7TYdS1i6sDjGiOJTWFDO9T8Zw5j39uGNKV0AWLGmIa2brWfZ6ka0braeFWuiH19LVzamXcu1W85t23IdS1c2jiXu2lJYUMaNPx7N6IldeX3q3lvKT+o9gyP2n8Nld55Mvn1pGexMH+NYZDPK0m2u3yBlfVP4W0bNXwoCxlQ8HWNm3c1scBqvvyllveJ1dvZaW5jZUDPrZWa9iqm/I6dmzYwpjejUZTPtO2+iqLic/gNX8s7o5nGHFQPjt+e+zpxFLXni1QO3lL41dU9O7PspACf2/ZTxU/cEYPyHe3JCn5mA0X2vxazbUC/PmyuMq374OnMWt+DxcVs/nz77fcnZx0zht/ecyKaSfPwiF2VpLnHLZg15MdBOUmtgLXAyMKqa498AzgZeldQDqPgX8w5wp6R9zWxWaFDvZGaf7kRMlV6LqM16L0n7mNlnRHdQc0Z5mbjz95246dHPKSiE0Y+1Ys6nDWo+Mc8csM9iTuwzi8/mt+K+q54G4J6Rh/HI6IO4dvBYvnv4DBYtb8LV90ZNfe981Jl++89l+LWPs2lzEX966Ntxhp91B3ZZxIDDZjJrQSse+HXUIen/XujN5ae9RXFRGbde9CIA02a3469PHgXAU398hMb1SygqKuNbB8zmiru/y+zFLWN7Dzsjl2rIWUvIZlYi6TrgPaJ+d5/UcMrdwL8lfQx8TNScgZl9FW7SDZdUUSX9A7DDCbmqa5nZp6Ep4kVJ64E3gaY7ev04vf9qM95/tVncYcTqw886cNRFP6l03xW3fbeSUnHL40dkN6gEmfpFR464/Kfblb994x5VnnP6dZXdnsk9Saj9piOrbchmdhuhXbiK/UsJbchmtoGo50Rlx70KHFZJ+fkp67OBHmH9AeCBlH0np3GtUURtyc65PGImryE751wSGCSiB0U6PCE75/Kcz6nnnHOJEN3U8zZk55xLhEw+qZdNnpCdc3nNkNeQnXMuCcz8pp5zziWG15Cdcy4BoiYLb0N2zrlE8Cf1nHMuAbzbm3POJYY3WTjnXCJEvSxyIyHnRpTOObcLyq0grSUdklpIekrSJ5I+ltRPUitJYyTNDH9bhmMl6TZJsyRNldSzumt7QnbO5bWKB0PSWdL0T2CUme1HNAfox8CVRLMUdQXGhm2AAUDXsAwhGma4Sp6QnXN5rxyltdREUnOi6ebuAzCzzWa2EhgIDAuHDQNOCesDgQct8g7QQlLHqq7vCdk5l9cqelmkWUNuI2lCyrLt3JpdgK+IJtOYLOneMPNQezNbGI5ZBLQP652AuSnnzwtllfKbes65/GaitDztR6eXmlmvavYXAT2BS83sXUn/ZGvzRPRyZibJdiZUryE75/KakbkmC6Ia7jwzezdsP0WUoBdXNEWEv0vC/vlA55Tzdw9llfKE7JzLe5m6qWdmi4C5krqFomOB6cBIYFAoGwSMCOsjgfNCb4u+wKqUpo3teJOFcy6vZeFJvUuBRyTVAz4HLiCq3D4haTAwB/hBOPYl4CRgFrA+HFslT8jOubyXyYRsZlOAytqZj63kWAMuTvfanpCdc3nNB6h3zrmkMCjNkUenPSE75/Kaj/bmnHMJ4gnZOecSwNuQnXMuQcwTsnPOJUOaT+HFzhOycy6vmUFZufeycM65BPA2ZOecSwxvQ3Z1XtPJVY6h4gJbsSruEBKtcO2GXb6G90N2zrmksKgdORd4QnbO5TUDyvzRaeecSwK/qeecc4nhTRbOOZcQ3svCOecSwMwTsnPOJUautCHnxq1H55zbBeXlSmtJh6TZkj6UNEXShFDWStIYSTPD35ahXJJukzRL0lRJPau7tidk51xeM4RZessOONrMDjazirn1rgTGmllXYGzYBhgAdA3LEODu6i7qCdk5l/cszWUXDASGhfVhwCkp5Q9a5B2ghaSOVV3EE7JzLr+Fm3pp1pDbSJqQsgyp/IqMljQxZX97M6sYK2AR0D6sdwLmppw7L5RVym/qOefyX/rV36UpzRBVOdLM5ktqB4yR9MnXXsrMJO1UhdtryM65vJfJNmQzmx/+LgGeBXoDiyuaIsLfJeHw+UDnlNN3D2WVqrKGLOl2qvleMbPL0oreOediZJB2D4qaSGoMFJjZmrD+HeA6YCQwCLg5/B0RThkJXCLpMaAPsCqlaWM71TVZTMhA/M45Fy8DMtcPuT3wrCSI8uejZjZK0vvAE5IGA3OAH4TjXwJOAmYB64ELqrt4lQnZzIalbktqZGbrd/ZdOOdcXDI1loWZfQ4cVEn5MuDYSsoNuDjd69fYhiypn6TpwCdh+yBJd6X7As45F7ta6PeWCenc1LsVOAFYBmBmHwBHZTEm55zLoKw8GJIVaXV7M7O5oc2kQll2wnHOuQwzsAzd1Mu2dBLyXEmHAyapGPg58HF2w3LOuQxKQHNEOtJpsriQqFG6E7AAOJgdaKR2zrn4Kc0lXjXWkM1sKXBOLcTinHPZkS81ZEl7S3pe0leSlkgaIWnv2gjOOecyIo96WTwKPAF0BHYDngSGZzMo55zLmIoHQ9JZYpZOQm5kZg+ZWWlYHgYaZDsw55zLFCtPb4lbdWNZtAqrL0u6EniM6LvmTKLHAZ1zLjckoPabjupu6k0kSsAV7+SnKfsMuCpbQTnnXCbt3GCYta+6sSy61GYgzjmXFQm5YZeOtJ7Uk9QD6E5K27GZPZitoJxzLnOSccMuHTUmZElXA/2JEvJLRJP2jQc8ITvnckOO1JDT6WVxOtGwcovM7AKioeeaZzUq55zLpPI0l5il02SxwczKJZVKakY0NUnnmk5ytatX/9VceP0CCguMl4e34ok72td8Uh7rtMdarrx+0pbtDp3W8/A936B12430PnIxpSUFLJzfiFtvOJh1a4tjjDQexfXK+cvDH1BczygsNMaPbsMjt+/Jr//6CV17rKW0RHz6YVNuv3pfykpzfKa3zA5Qn1XpfNITJLUA7iHqeTEJeDubQVWQ9LtdPP+lEPuOnne+pDt25bVrU0GBcfFN8/nDOV34Sf9uHD1wJXt03Rh3WLGa/2UTLh10FJcOOoqfX/AtNm0s5L+vd2Dye2256Jxvc8m532bBl034wXmz4g41FiWbxVXnH8glp/TkklMPodeRK+h20Gpee74dQwYcykXf70m9BuWccPqiuEPNCFl6S9xqTMhmdpGZrTSzfwHHA4NC00Vt2KWEbGYnmdnKDMWSWN0OWc+C2fVY9GV9SksKGDeiBf1OWBV3WIlxUK+lLJzfiK8WNWLye20pL4v+2X8yrQWt222IObq4iI3rCwEoKjIKi8rBYMIbragYaOfTqU1p02FzrFFmTAYfnZZUKGmypBfCdhdJ70qaJelxSfVCef2wPSvs36uma1eZkCX13HYBWgFFYT2jJD0naaKkaZKGSLoZaChpiqRHJF0n6fKU42+U9HNJ/SW9IelFSTMk/UtSQThmtqQ2Yf08SVMlfSDpoVD2vfBBTZb0H0k5+Tu/dYcSvlpQb8v20oXFtOlYEmNEyXLU8Qt4fcxu25Uff/JcJr7dLoaIkqGgwLj92Uk8+tY7TP5vS2ZMbbZlX2FROcd8fzET32wZY4SJte0QxH8GbjGzfYEVwOBQPhhYEcpvCcdVq7o25L9Xs8+AY2q6+A76sZktl9QQeB/4NnCJmR0MEL5dngFuDQn3LKLptw8If7sTTS44CjgNeKriwpL2B/4AHG5mS1OeQhwP9DUzk/S/wG+AX1YXpKQhwBCABjTKwNt22VRUVE6fIxcx7K79vlZ+5qCZlJWJ117pFFNk8SsvF5ee2pPGTUv5wx3T2bPrOubMbAzAxX/8jI8mNGfaxPy4f6/MzTq9O/Bd4EbgF4pm7jgGODscMgy4BrgbGBjWIcpHd0hSmGevUtU9GHL0rga/gy6TdGpY7wx03Sae2ZKWSTqEaObXyWa2LMxk8l6YfBBJw4EjSUnIRB/Yk2EoUcxseSjfHXhcUkegHvBFTUGa2VBgKEAztUpAqxMsW1RM2922/rRs07GEpQvr3o2qyvTqt4TPZjRn5Yr6W8qOO2kuhx2xmN9f2o8kjIEbt3Vripj6bnMO/dYK5sxszNkXz6F5qxJuv/SbcYeWGZl9MORWoopb07DdGlhpZqVhex7R2PGEv3MBzKxU0qpw/NKqLp6I26eS+gPHAf3M7CBgMpUPYHQvcD7RVNr3p5Rv+3Gn+/HfDtxhZgcQPRqek4MmzZjSiE5dNtO+8yaKisvpP3Al74zOj5rNroqaK7bWgg/tu4T/+dFnXPebw9i0qTDGyOLVrOVmGjeNcki9+mUccvhK5n3ekBNOX0TPI1fw5192S8QccxmTfhtyG0kTUpYhFZeQdDKwxMwmZivMtJ7UqwXNidpa1kvaD+gbykskFZtZRYPos8B1QDFbfyIA9JbUhajJ4kxCDTbFq8Czkv4RatWtQi25OTA/HDMo82+rdpSXiTt/34mbHv2cgkIY/Vgr5nyak98tGVW/QSmH9P6KO/58wJayC3/5EcXF5dz4z3eB6MbenX85MK4QY9OqbQm/vHkGBYWGBG+OasN741rz/EdvsmRBA/7+2AcA/HdMa4bftWfM0e66HehBsdTMelWx7wjg+5JOIqq8NQP+CbSQVBRqybuzNafMJ/q1P09SEVG+WVbdiyclIY8CLpT0MTADeCeUDwWmSppkZueY2WZJrxH9REidaPV94A5gX+A1osS9hZlNk3Qj8LqkMqIa+PlE7TtPSlpBlLRzdvyO919txvuvNqv5wDpk08YifnjiCV8r+8kZmb71kZtmf9qYS0/b/t7893p8K4ZoakEGmizM7CrCoGrhV/2vzOwcSU8SPUD3GFHFbkQ4ZWTYfjvsf7W69mNI79FpEU3htLeZXSdpD6CDmb23M2+qMma2ieiR7G2NA36bEksBUe35jG2OW21mJ1dy3b1S1ocRNbin7h/B1g8vtfwB4IE0w3fOJV127/b8FnhM0g1Elb37Qvl9wEOSZgHLiToiVCudGvJdRA8VHkPUXLAGeBo4bMfj3nmSugMvAM+a2czafG3nXO6SZa6XRQUzG0dUYSR0KOhdyTEb2b7yWK10EnIfM+spaXJ4kRUVHZ9rk5lNB7abyy/1g3HOuUoloj9UzdJJyCWSCglvSVJbEjEMh3POpScJj0WnI51ub7cR3SRrF26MjQduympUzjmXSTky63SNNWQze0TSRKIhOAWcYmYf13Cac84lQ0IGDkpHOr0s9gDWA8+nlpnZl9kMzDnnMiZHGlnTaUN+ka2TnTYg6qs7A9g/i3E551zG5E0NOTxWvEUY6e2irEXknHN11A4/qWdmkyT1yUYwzjmXFflSQ5b0i5TNAqAnsCBrETnnXCbl0009tg4zB1BK1Kb8dHbCcc65LMiHhBweCGlqZr+qpXiccy6jBCjXe1lUDCcn6YjaDMg55zIuD2rI7xG1F0+RNBJ4ElhXsdPMnslybM45t+vyrA25AdGgysewtT+yEc1v55xzyZcHCbld6GHxEVsTcYUceXvOOUfOZKzqEnIh0ITKZ4HMkbfnnHP50WSx0Myuq7VInHMuG4y8GMsij6acdc7VZblSQ65uPORjay0K55zLpgyNhyypgaT3JH0gaZqka0N5F0nvSpol6fGKWZUk1Q/bs8L+vaq7fpUJ2cyWp/M+nXMu6WTpLWnYBBxjZgcBBwMnSuoL/Bm4xcz2BVYAg8Pxg4EVofyWcFyV0pkxxDnncluGasgWWRs2i8NiRN2Cnwrlw4BTwvpAts52/xRwrKQqm4M9ITvn8lq6teN025klFUqaAiwBxgCfASvNrDQcMg/oFNY7AXMBwv5VQOuqrr3Dw28651zOSf+mXhtJE1K2h5rZ0K9dyqwMOFhSC6L5RvfLRIjgCdk5VwfsQC+LpWbWK50DzWylpNeAfkCLivF/gN2B+eGw+UBnYJ6kIqA50ZPPlfImC+dc/stcL4u2oWaMpIbA8cDHwGvA6eGwQcCIsD4ybBP2v2pmVb6S15Cdc/kvc/2QOwLDwtDEBcATZvaCpOnAY5JuACYD94Xj7wMekjQLWA6cVd3FPSE75/JbBkd7M7OpwCGVlH8O9K6kfCNwRrrX94TsnMt7OT9AvXPO5Y0ceXTaE7LLmrK2zeMOIfFGvf183CEkWu8T1tZ8UBpyZSwLT8jOufyWZg+KJPCE7JzLf56QnXMufsKbLJxzLjFUnhsZ2ROycy6/eRuyc84lhzdZOOdcUnhCds65ZPAasnPOJYH5o9POOZccXkN2zrn4eT9k55xLkqrHhE8UT8jOubznNWTnnEsCfzDEOeeSQ2VxR5Aen+TUOZf3ZOktNV5H6izpNUnTJU2T9PNQ3krSGEkzw9+WoVySbpM0S9JUST2ru74nZOdcfjOim3rpLDUrBX5pZt2BvsDFkroDVwJjzawrMDZsAwwAuoZlCHB3dRf3hOycy3uZqiGb2UIzmxTW1wAfA52AgcCwcNgw4JSwPhB40CLvAC0kdazq+p6QnXP5z9JcdoCkvYhmoH4XaG9mC8OuRUD7sN4JmJty2rxQVim/qeecy2s7+GBIG0kTUraHmtnQ7a4pNQGeBi43s9WStuwzM5N2rqOdJ2TnXH4z25EB6peaWa/qDpBUTJSMHzGzZ0LxYkkdzWxhaJJYEsrnA51TTt89lFXKmyycc/kvQ00WiqrC9wEfm9k/UnaNBAaF9UHAiJTy80Jvi77AqpSmje14Ddk5l/cy+KTeEcC5wIeSpoSy3wE3A09IGgzMAX4Q9r0EnATMAtYDF1R3cU/Izrn8ZkCG5tQzs/FEzdKVObaS4w24ON3re0J2zuU/f3TaOeeSwWedds65hPDR3pxzLgl8tDfnnEuG6MGQ3MjInpCdc/nPJzl1zrlk8Bqyq1W9+q/mwusXUFhgvDy8FU/c0b7mk/LMFZe9TZ9e81m5qgEXXnoyAP97/iT69J5PaWkBCxY24R+39WPdunq0b7eWoXe+wLz5zQD4ZEZrbr+7T5zh14pnhrbl5UdbIUGX/Tbyy1u+5OVHW/PsvW1ZOLs+T3z4Ic1bR6O5fzmzPv/4xR7M+rAhg367kDN+9lXM0e8ks4z1Q862RCdkSS2As83srp08/79mdvhOnHcNsNbM/rYzr1vbCgqMi2+az1Vn7c3ShcXc/tJM3nmlOV/ObBB3aLVqzNi9ef6Fbvzqiv9uKZs0pSP3P3gw5eUF/HjQZM48fRr3DzsEgIWLmnDx5SfFFW6tW7qwmOfua8M94z6hfkPjhp/uybgRLdn/sHX0OX41v/mffb92fLOWZfzs+nn8d1TzmCLOnFzpZZH0sSxaABft7Mk7k4xzUbdD1rNgdj0WfVmf0pICxo1oQb8TVsUdVq37aFp71qyt97WySVM6Ul4e/TP/ZEYb2rReH0doiVFWKjZtLKCsFDZtKKB1+xL2PWADHTpv3u7YFm1K6XbwBooSXW1LU+YGqM+qpCfkm4F9JE2R9KSkUyp2SHpE0kBJ50saIWlcmD7l6pRj1qas/1bSh5I+kHRzKPuJpPdD2dOSGtXmm8uU1h1K+GrB1kS0dGExbTqWxBhRMn3nuM+YMGm3Ldsd2q/ljltf4i83jWH/7kuqOTM/tOlYwuk/W8K5h3Xnhwf3oHHTMg7tvybusLLPQOXpLXFLekK+EvjMzA4G7gDOB5DUHDgceDEc1xv4H+BA4AxJXxs+T9IAopH7+5jZQcBfwq5nzOywUPYxMDir78bF5qwzPqKsTLw6bi8Ali9vyLmDT+WSy09i6H09ufKXb9GoYX5/ia1ZWcjbrzRn2LvTeXTyR2xcX8jYp1vGHVbt8BpyZpnZ60BXSW2BHwJPm1lp2D3GzJaZ2QbgGeDIbU4/Dvi3ma0P11oeyntIelPSh8A5wP41xSFpiKQJkiaUsCkD72zXLVtUTNvdtv7kbNOxhKULi2OMKFmOP+Yz+hw2n7/8/QgqxoUpKS1kzZr6AMz6rDULFzWhU6fVMUaZfZPfbEKHzptp0bqMomI44qSVTJ/QOO6wakcWZgzJhpxJyMGDwI+IhrC7P6V8248y3Y/2AeASMzsAuBao8S6YmQ01s15m1quY+mm+THbNmNKITl02077zJoqKy+k/cCXvjM79GzGZcGjPBZx+2nSuueHbbNq8tTG0ebONFBREv1E7tF/DbrutYeGiJnGFWSvadSrh40mN2LhemMGU8U3ZY9+NcYdVK1RentYSt6Q3168BmqZsPwC8Bywys+kp5cdLagVsIJpc8MfbXGcM8EdJj5jZekmtQi25KbAwzABwDtWM5J9k5WXizt934qZHP6egEEY/1oo5n9atHhYAV/5qPAf2WEyzZpt46P5neHj4gZx5+jSKi8q56bpXga3d23rsv4TzzplKaWkBZnD7Xb1ZuzYZX7DZsl/P9Xzru6u4+IRuFBYZ+/bYwIAfLeO5e9vw5N3tWL6kmAuP24/ex6zmir/PZfmSIi4d8A3WrylEBfDcvW0ZOu4TGjeNP3HtECNnHgyRJaDdpDqSHiVqG37ZzH4taRTwnJn9K+w/nygJNyeaHuVhM7s27FtrZk3C+pXAecBm4CUz+52knwG/Ab4imqiwqZmdn263t2ZqZX203RCoLlCvHnGHkHijRj4cdwiJ1vuEuUz4YGNV4w+npXnj3axv95+mdezoCddMrGkKp2xKeg0ZMzu7Yj30gugKDN/msHlmdkol5zZJWb+ZqNdG6v67gbsrOe+aXQraOZcsCa94VsiZNmRJxxH1hLjdzOpeJ1vn3M7zXhaZZWb/MbM9zezWbcofMLNLYgrLOZd0BiqztJaaSLpf0hJJH6WUtZI0JjwHMUZSy1AuSbdJmiVpqqSeNV0/ZxKyc87ttMzVkB8ATtym7EpgrJl1BcaGbYABRE2sXYEhVNI8ui1PyM65PJdmMk4jIZvZG8DybYoHAsPC+jCiTgYV5Q9a5B2ghaSO1V0/8Tf1nHNulxg70j7cRtKElO2hZja0hnPam9nCsL4IqBhqsRMwN+W4eaFsIVXwhOycy3/p90Neuivd3szMpJ0fW86bLJxzeU9maS07aXFFU0T4WzFS1Xygc8pxu1PDw2eekJ1z+c2AsvL0lp0zEhgU1gcBI1LKzwu9LfoCq1KaNirlTRbOuTyXuT7GkoYD/YnamucBVxM9cPaEpMHAHOAH4fCXgJOAWcB6ojF4quUJ2TmX/zKUkM3sh1Xs2m4MBYvGpbh4R67vCdk5l/8S8BReOjwhO+fym+GTnDrnXDIYlJfFHURaPCE75/Kb15Cdcy5BvA3ZOecSwhOyc84lQTLGOk6HJ2TnXH4zIAETmKbDE7JzLv95QnbOuSQw72XhnHOJYGDmNWTnnEsGryE751xCeC8L55xLADO/qeecc0lhZT6WhXPOJYA/GOKcc8nggws551yC5Ei3N5/k1DmX1wywcktrSYekEyXNkDRL0pWZjNVryM65/GaWsZt6kgqBO4HjgXnA+5JGmtn0TFzfa8jOufxn5ektNesNzDKzz81sM/AYMDBTYcpy5O5jEkn6imja76RoAyyNO4iE88+oekn7fPY0s7a7cgFJo4jeVzoaABtTtoea2dCUa50OnGhm/xu2zwX6mNkluxJjBW+y2AW7+g8l0yRNMLNecceRZP4ZVS8fPx8zOzHuGNLlTRbOOZe++UDnlO3dQ1lGeEJ2zrn0vQ90ldRFUj3gLGBkpi7uTRb5ZWjNh9R5/hlVzz+faphZqaRLgFeAQuB+M5uWqev7TT3nnEsIb7JwzrmE8ITsnHMJ4QnZOecSwhOyc84lhPeyyFGSnicaN6VSZvb9WgwncSTdTvWfz2W1GE7iSPpFdfvN7B+1FYvbyhNy7vpb+Hsa0AF4OGz/EFgcS0TJMiHuABKuadwBuO15t7ccV9mjrvn4+KtzdYHXkHNfY0l7m9nnAJK6AI1jjikxJLUFfgt0Jxo4BgAzOya2oBJEUgNgMLA/X/98fhxbUHWY39TLfVcA4ySNk/Q68BpwebwhJcojwMdAF+BaYDbR468u8hBRk9cJwOtEYzOsiTWiOsybLPKApPrAfmHzEzPbFGc8SSJpopkdKmmqmR0Yyt43s8Piji0JJE02s0MqPh9JxcCbZtY37tjqIq8h5zhJjYBfA5eY2QfAHpJOjjmsJCkJfxdK+q6kQ4BWcQaUMBWfz0pJPYDmQLsY46nTvA059/0bmAj0C9vzgSeBF2KLKFlukNQc+CVwO9CMqJnHRYZKagn8gWjUsibAH+MNqe7yJoscV9GjouKnZyj7wMwOijs259yO8SaL3LdZUkPCQxCS9gG8DTmQNExSi5TtlpLujzGkRJF0UyWfzw0xhlSneULOfVcDo4DOkh4BxgK/iTekRDnQzFZWbJjZCuCQ+MJJnAGVfD4nxRdO3eZtyDnOzMZImgT0BQT83MySNEll3AoktQyJBkmt8H/3qQol1a/omRN+bdWPOaY6y/9h5ocGwAqi/5/dJWFmb8QcU1L8HXhb0pNEX1inAzfGG1KiPAKMlfTvsH0BMCzGeOo0v6mX4yT9GTgTmAaUh2Kr64MLpZLUHah4Mu9VM5seZzxJI2kAcGzYHGNmr8QZT13mCTnHSZpB1E7qN/JSSGpmZqtDE8V2zGx5bcfkXE28ySL3fQ4U4z0rtvUocDJRH20jaq6oYMDecQSVFJLGm9mRktbw9WFKRfQLq1lModVpXkPOcZKeBg4i6l2xJSnX9fF+nctFXkPOfSPD4qog6TTgSKKa4Jtm9ly8ESWLpJ5s/XzGm9nkmEOqs7yG7PKapLuAfYHhoehM4DMzuzi+qJJD0h+BM4BnQtEpwJNm5g+HxMATco6S9ISZ/UDSh1QyVVHFyGZ1naRPgG9a+IcuqQCYZmbfjDeyZAg3hQ8ys41huyEwxcy6xRtZ3eRNFrnr8vDXR3ar3ixgD2BO2O4cylxkAVE/9o1huz7RAFUuBl5DzlGSJplZT0kPmdm5cceTVGHQ/sOA94h+SfQmmm9vFfhksJKeI/p8xhB9PscTfVbzwG8O1zavIeeuepLOBg4PN62+xsyeqeScusiHkqzes2GpMC6mOByekHPZhcA5QAvge9vsM7bepKnrDgQerhjLwm1nOfCimZXXeKTLOk/IOcrMxgPjw3jI98UdT4K1B94PAzDdD7xi3k6X6kzg1tCf/X4z+yTugOoyb0POcZIaE82AsYeZDZHUFehmZj5jSCBJwHeIBs7pBTwB3Gdmn8UaWEJIagb8kOjzMaJZaIabmU92Wst8POTcdz+wGTg8bM8HvA9pilAjXhSWUqAl8JSkv8QaWEKY2WrgKeAxoCNwKjBJ0qWxBlYHeQ05x/kUTtWT9HPgPGApcC/wnJmVhP7IM81sn1gDjJmkgcD5RA/PPAgMM7MlYfLc6Wa2V4zh1Tnehpz7fAqn6rUCTjOzOamFZlbus3MDcDZwS+r42ZL+bGa/lTQ4xrjqJG+yyH0+hVP19t42GUt6CMDMPo4npETpWslkBgMAzGxsDPHUaV5DznE+hVON9k/dkFQEHBpTLIkh6WfARcDekqam7GoKvBVPVM7bkHNUGKGrSmY2qbZiSSJJVwG/AxoC61N2lQBDzeyqWAJLCEnNiW5u/gm4MmXXGh+8Pz6ekHOUpNfCagOirlwfENWQDwQmmFm/uGJLEkl/Av4CfIPos4Ko44XPOegSx5sscpSZHQ0g6Rmgp5l9GLZ7ANfEGFrSfA68AewOTCFq2nmbrXPsOZcYflMv93WrSMYAZvYR4ENLbnUZ0eA5c8KX2CHAylgjcq4KXkPOfVMl3Qs8HLbPAaZWc3xds9HMNkpCUn0z+0SSj/XrEskTcu67APgZ8POw/QZwd3zhJM48SS2A54AxklawdWxk5xLFb+q5OkPSt4HmwCgz2xx3PM5tyxNyjpN0BNFNvD1J+cVjZnV6mnvncpEn5BwX5oy7ApgIlFWUm9my2IJyzu0Ub0POfavM7OW4g3DO7TqvIec4STcDhUQzhGwZVKiuP6nnXC7yhJzjUp7Yq/gfKaIn0fzBB+dyjDdZ5L5xlZT5t6xzOcgTcu5bm7LeADgZ8GElnctB3mSRZyTVJ5rIs3/csTjndoyPZZF/GhENpOOcyzHeZJHjJH3I1jbjQqAtcF18ETnndpY3WeQ4SXumbJYCi82sNK54nHM7zxOyc84lhLchO+dcQnhCds65hPCE7LJGUpmkKZI+kvSkpEa7cK0HJJ0e1u+V1L2aY/tLOnwnXmO2pDbplm9zzNrq9ldy/DWSfrWjMbr85gnZZdMGMzvYzHoAm4ELU3dK2qlePmb2v2Y2vZpD+gM7nJCdi5snZFdb3gT2DbXXNyWNBKZLKpT0V0nvS5oq6acAitwhaYak/wDtKi4kaZykXmH9REmTJH0gaaykvYgS/xWhdv4tSW0lPR1e4/0whjSSWksaLWlamAZLNb0JSc9JmhjOGbLNvltC+VhJbUPZPpJGhXPelLRfRj5Nl5e8H7LLulATHgCMCkU9gR5m9kVIaqvM7LDwlOFbkkYTTUbaDegOtAemA/dvc922wD3AUeFarcxsuaR/AWvN7G/huEeBW8xsvKQ9gFeIJoK9GhhvZtdJ+i4wOI238+PwGg2B9yU9HcaebgxMMLMrJP0xXPsSYChwoZnNlNQHuAuf8dpVwROyy6aGkqaE9TeB+4iaEt4zsy9C+XeAAyvah4mmWOoKHAUMN7MyYIGkVyu5fl/gjYprmdnyKuI4DugubakAN5PUJLzGaeHcF8N8ezW5TNKpYb1ziHUZUA48HsofBp4Jr3E48GTKa9dP4zVcHeUJ2WXTBjM7OLUgJKZ1qUXApWb2yjbHnZTBOAqAvma2sZJY0iapP1Fy72dm6yWNIxrQqTIWXnfltp+Bc1XxNmQXt1eAn0kqBpD0DUmNiWbPPjO0MXcEjq7k3HeAoyR1Cee2CuVrgKYpx40GLq3YkHRwWH0DODuUDQBa1hBrc2BFSMb7EdXQKxQAFbX8s4maQlYDX0g6I7yGJB1Uw2u4OswTsovbvUTtw5MkfQT8H9Evt2eBmWHfg8Db255oZl8BQ4iaBz5ga5PB88CpFTf1gMuAXuGm4XS29va4liihTyNquviyhlhHAUWSPgZuJvpCqLAO6B3ewzFsHU/kHGBwiG8aMDCNz8TVUf7otHPOJYTXkJ1zLiE8ITvnXEJ4QnbOuYTwhOyccwnhCdk55xLCE7JzziWEJ2TnnEsIT8jOOZcQ/x+ZdtBY3VRV+QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ConfusionMatrixDisplay(confusion_matrix=cm1, display_labels=[x for x in categories_l1.values()]).plot(xticks_rotation='vertical')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "1606f1e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7f1404828430>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaUAAAFxCAYAAADTfbeXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABLw0lEQVR4nO3dd5hU5dnH8e9vC7BLlSIdAbGjAmIvQVGxYNBY0SSamBi7JjGWaN5US4oliiUYNYrd2BMD2FDsgKCgSO+gsvTOlvv945yBYZmdnYXdPWeG+3Ndc3H6ufcAc+9TzvPIzHDOOefiIC/qAJxzzrkET0rOOediw5OSc8652PCk5JxzLjY8KTnnnIsNT0rOOediw5OSc865jEmaLWmipAmSxobbWkp6XdK08M+dwu2SdLek6ZI+l9Sn2uv7e0puWzVQQ2tE46jDyIgaNow6hJrJU9QR1EhZcX7UIWSsrCjqCGpm4/z5JWbWZlvPH3B0Y1uytDyjY8d9vmGEmZ2Q7hhJs4G+ZlaStO0vwFIzu03S9cBOZnadpJOAK4CTgIOBv5vZwemuX5BRpM6l0IjGHKz+UYeRkfyuu0YdQo1YUYOoQ6iRJb1bRB1CxpbtHXUENTPzV7+csz3nlywt5+MRnTI6trD9jNbbeJtBQL9w+VFgFHBduP0xC0o/H0lqIam9mS2q6kJefeeccznNKLeKjD5Aa0ljkz4XpbwgjJQ0Lml/26RE8zXQNlzuCMxLOnd+uK1KXlJyzrkcZkAFGTfTlJhZ32qOOcLMFkjaGXhd0ldb3M/MJG1zu5AnJeecy3EVVNTatcxsQfjnt5JeBA4CvklUy0lqD3wbHr4A6Jx0eqdwW5W8+s4553KYYZRbZp/qSGosqWliGTgemAS8ApwfHnY+8HK4/Arww7AX3iHAinTtSeAlJeecy2kGlNZeSakt8KIkCPLHk2Y2XNIY4FlJFwJzgLPC418j6Hk3HVgL/Ki6G3hScs65HFeDNqW0zGwmsH+K7UuArbrihr3uLqvJPTwpOedcDjPIqGouLjwpOedcjqu9bg51z5OSc87lMMMor6Xqu/rgSck553KZQXn25CRPSs45l8sMUUr2jKXoSck553KYARVeUnLOORcX5V5Scs45FweGJyXnnHMxUmGelJxzzsVABWIj2TMJoyclFxt9+63k4j8uJD/P+N9TLXl2SNvqT6pnjzw9nHVrCyivEBXl4qqfHQPAKafNYOBpM6goF2M+asfD/9g34kihceONXH31J+yyywrMxJ13HsSGDflcccVYGjUq49tvG/OXvxzK2rWFkcR30/fe5og957BsTRGD/342AD/tP4ZBfSezfE0wPex9Iw/ig6m7MGD/qfzgyM82nduj3RJ+cO8ZTFu0rXPS1Uy74tX89bC3aN1oHQY8M20vHp2y36b9P97rM2444EMOeu58lm0oon+nWVy9/xjMRJnlcfPYwxi3uH29xJqKl5TcdpM0CrjGzMamOeZI4AGgFDiZYKrhM2rh3v3Cew/c3mtlKi/PuOyWBdxwTndKFhVyz2vT+GhEc+ZOa1RfIWTs+p8fycoVm6dX36/XYg45YiGXXdifstJ8mrdYH2F0m1188aeMHduem28+goKCcho2LOeWW0bxz3/2YuLEnTn++Jmcfvpkhg3br/qL1YH/froHz33Uk9+d+dYW2596fz+eeK/XFttGfLY7Iz7bHYBd2y7hr98fUW8JCaDcxK2fHsqXS9vQuGAjL570PO9/3YnpK1rSrng1R7Sfx4LVTTYd/+HXnXhzfldA7NFiCX8/8nVOePWceos3Wba1KfnUFdntPOBWM+tlZgtqIyFFZY/ea1k4uwFfz21IWWkeo15uwaEDVkQdVkZOHjST557cg7LSoIpkxfLoE2lx8UZ69lzMiBHdASgry2fNmgZ07LiKiRPbAPDpp2054oj5kcU4fnYHVq5tWP2BlRy//3Re/7x+p7dfvK4xXy4NntuasgbMWLETbYvWAHDjAR/wl08P2WLMhLVlhRAmgqKC0nqNdWui3PIy+sSBl5TqiKSuwH/MrGe4fg3QhGAe+4+Bo4EWwIVmNlpSEfAIwQi8XwFFSdc6Hvg90BCYQTD8+zkEw8MPkHQicGPifpIuAL4LFAO7Ai+a2bVVXcvMVks6AbiLYHj59+rimaTTql0pixc22LResqiQPfusre8wqmUGf/rre5iJ/73ajeH/6UaHzqvZZ98Szr/wCzZuzOef9/dk2pSWkcbZrt0aVqxoyC9+8THduy9n2rSWPPBAH+bMacahhy7gww87ceSR82jdOn7P+MxDJ3FS76lMXtCGv792GKvWb5m4jtt3Btc8fkJE0UHHxivZu2UJny1pS/9Os/hmXTFfLd+61HZc51n8stfHtGq0jp++fWIEkQaCmWfjkXAykT2R5pYCMzsIuBr4bbjtEmCtme0VbjsAQFJr4CbgWDPrA4wFfmFm/ySYQOtXZnZeinv0As4G9gXOltS5qmtJagQ8CJwS3rddVYFLukjSWEljS9mwPc8gK/3qiu9w5UX9+b/rDmPgqTPpuV8J+flG02Yb+fml/XjogZ7c8LtPIOKxxvLzjR49lvHf/+7G5ZefwPr1BZx11pfceefBDBw4jbvvHkFRURllZfH6Cnj+43343t/O5ftDzmTJqmKuOumDLfbv0+kb1pcWMPObaJJ+cUEpQ44ayc1jD6OsQlzSczx3fXZgymNfn9eNE149h0vfGcDV+4+p50i3VI4y+sRBvP5F7jheCP8cB3QNl48CHgcws8+Bz8PthwB7A+9LmkAwq+MuGdzjTTNbYWbrgS/Dc6q61p7ALDObFs5/8nhVFzWzoWbW18z6FlLzqpeqLPm6kDYdNm5ab92+lJJF0TTAp7OkJCjArljeiA/fa8/uey2lZHEjPni3IyCmftUSqxDNmm9Mf6E6VlJSRElJEVOmtALgvfc60aPHMubPb8aNNx7NlVcO4J13urBoUZNqrlS/lq4upsLyMBMvjdmLfTp/u8X+4/ebzsjPekQSW4HKGXLUCF6ZvRsj53WnS9OVdGqykldPfo63T32cdsVreOmk52ndaMvS55hvO9ClyUp2argukrjNRKnlZ/SJA09KdaeMLZ9vckNDoohRTvVVqAJeD9uNepnZ3mZ2YQb3Ty7GJO6zrdeqc1MmFNOx20badt5AQWEF/QYt56ORzaMOawsNG5VRVFS6abl332+ZM6sZH73Xgf16LwagY6dVFBRWsHJFg3SXqnPLlhWxeHExHTuuBKBXr2+YO7c5zZsHnTAk45xzvuS116L5gq9Kq6ZrNi3322cWM5JKRJLRf98ZjPw8ipiNWw59hxkrduKRycEcd1OXt+KQf1/A0S99n6Nf+j5fr23Mqa+dTsn6Yro0WUGitLx3y8UU5pezbEM0bY1BR4e8jD5x4G1KdecbYGdJrYDVwEBgeJrj3wXOBd6S1BNIdIn6CLhXUg8zmy6pMdDRzKZuQ0wpr0XQhtVV0q5mNgMYvA3X3i4V5eLeGztyy5MzycuHkU+3ZM7U6DsMJNtppw3c9MePAMjPr2DUm50Z90k7CgoquPq6cdz3yBuUlYo7bj0AYlAVcv/9B3DttR9SWFjBokVNuPPOg+nffxYDB04H4IMPOjFyZLfI4vvj2W9wQLeFtGi8nlevG8aDb/SlT/eF7N5+CWawaHlTbn3pqE3H9+66kG9WNGHhsmb1HusBbb7mtO5T+WpZS1456TkAbp9wEO8sTF1pcUKXmZzafSplFXmsLy/g6tHHEd2/CcWmE0MmZFk0I2G2kXQlcBWwAJgJzCbo6HCNmY0N23jGmlnXSh0dJhMki8vC444B/gyb6stuMrNXJP2LoHPDv5M7VoQdHfqa2eVhHP8B/mZmo9JcK7mjw2hg1+q6hDdTSztYW82AHEv5u9dvb63tZUXRlrRqaknvFlGHkLFle0cdQc3M/NUvx5lZ3209v8e+xXb7y7tndOypu362XfeqDV5SqkNmdjdwd5r9JYRtSma2jqBHXarj3gK2ak01swuSlmcDPcPlfwH/Sto3MGm5qmsNJ2hbcs7lmHJ/edY551wcGIpNe1EmPCk551wOM6DUsuerPnsidc45V2OGvPrOOedcfGTTiA6elJxzLoeZkVVdwj0pOedcThMVMXhvLlOelJxzLocZsNE7OjjnnIsDQz7Jn3POufjw95Scc87FggEV3tHBOedcPMRnrqRMeFJyzrkc5iUl55xzsZGY5C9beFJyzrkcl00vz2ZPpM4552rMgIrwBdrqPpmSlC9pfDhXG5K6SfpY0nRJz0hqEG5vGK5PD/d3re7anpSccy6nBTPPZvKpgasIJiNN+DNwp5n1AJYBF4bbLwSWhdvvDI9Ly6vv3A7BFn0bdQg1UrFqVdQh1MhOn0UdQeZ2ijqAGpq5necHHR1qr/edpE7AycDNwC8kCTgGODc85FHgd8D9wKBwGeDfwBBJsjRTnntScs65HFeDl2dbSxqbtD7UzIZWOuYu4FqgabjeClhuZmXh+nygY7jcEZgHYGZlklaEx5dUFYAnJeecy2GGKMu8912JmfWtaqekgcC3ZjZOUr9aCG8rnpSccy6HBVNX1Fr13eHAdyWdBDQCmgF/B1pIKghLS52ABeHxC4DOwHxJBUBzYEm6G3hHB+ecy3EVpow+1TGzG8ysk5l1Bc4B3jKz84C3gTPCw84HXg6XXwnXCfe/la49Cbyk5JxzOS0YJbzOyx/XAU9L+hMwHngo3P4QMEzSdGApQSJLy5OSc87luLoY+87MRgGjwuWZwEEpjlkPnFmT63pScs65HGaIsgofZsg551xM+HTozjnnYqGWe9/VOU9KzjmX43zqCuecc7EQ9L7zkpJzzrmY8DYl55xzsWDgve+cc87FRIajNcSFJyXnnMthiUn+soUnJeecy3FeUnLOORcLtT3JX13zpORio2+/lVz8x4Xk5xn/e6olzw5pG3VIW/j5LVM5qN8yli8p5JJT+gDQbY/VXPH7GTQqLufbBQ35yzV7sHZN/P5bxf3ZJvvFHXM5+NhVLC8p4GfH7BF1ONWK+7MNhhnKnveUsifSmJBULmmCpEmSXpXUItzeVdKkcLmfJJN0StJ5/0lMiiWpQNItkqaF15og6cYq7vdjSRMlfR7ec1C4/V+Szqh07OpK61dLWi+pedK2fpJWhPecLOm3KbYnPsfWwiPLSF6ecdktC7jpvG78tN8eHD1oOV12W19ft8/I6y+05aaf7LPFtqtvns4jt3fl0u/24YM3WnH6TxZUcXZ0suHZJhv5TEtuPK9b1GFkJFuebQXK6BMHnpRqbp2Z9TKzngRDsV9WxXHzgZSJBvgT0AHY18x6AUcChZUPktQpvMYRZrYfcAjweQ1iHQyMAb5Xafvo8L59ge9L6pO8PenzRg3utV326L2WhbMb8PXchpSV5jHq5RYcOmBFfd0+I5PGNmfVii1LQR27rmPimGYAfPr+ThxxfJWzPEcmG55tskkfN2HVsviVNlPJimdrtTefUn3wpLR9PmTzXPSVfQaskHRc8kZJxcBPgSvCYd0xs1Vm9rsU19gZWAWsDo9bbWazMglM0q5AE+AmguS0FTNbA4wDemRyzbrUql0pixc22LResqiQ1u1LI4woM3OmFXNo/6UAHHlCCa3bb4w4oq1l67PNBtnwbBNtSp6UcpykfKA/wcyKVbmZICkk6wHMNbNVGdzmM+AbYJakR5KrA0N/Ta5uq7TvHOBpYDSwh6StKroltSIofX0RbjqyUvXdrinOuUjSWEljS9mQwY+Q2+68cTcGnruIu58fT1Hjcso2xuM/tnPJsikpZUcZOV6KwgTQEZgMvF7VgWb2riQkHVHVMZJ+BFwFtAIOM7N5SeeXSzoBOJAgAd4p6YCkUtWvzOzfSddKblMaDJxmZhWSnieYaGtIuO9ISeOBCuA2M/sibO8abWYD0/3wZjYUGArQTC3TTmtcE0u+LqRNh82ljNbtSylZtFWNZuzMn1nMjRf2BIKqvIP6LY04oq1l67PNBtnwbLNt7DsvKdXcurA9ZhdAVN2mlFC5tDQd6CKpKYCZPRJebwWw1VggFvjEzG4lKP2cXl2AkvYFdgNelzQ7PC+5Cm+0mfU2swPM7IHqrlcfpkwopmO3jbTtvIGCwgr6DVrORyObV39ixJq3DL6QJOOcS+by2tPtIo5oa9n6bLNBtjzbcsvL6BMHXlLaRma2VtKVwEuS7ktz3EhJfwTaJ533EDBE0s/MbH1YFdig8rmSOgDtzOzTcFMvYE4G4Q0GfhcmssS1ZknaJdOfr75VlIt7b+zILU/OJC8fRj7dkjlTG0Ud1hauu/0r9jtoBc12KmPYO58w7J4uFBWXM/DcRQB88HprRj4fr+7AkB3PNtn1981hv0NX07xlGY+P/ZJht7dlxFOtog4rpWx4tmb+ntIOw8zGS/qcIAmMTnPozcDLSes3An8EJklaBawDHgUWVjqvEPhbmJzWA4uBizMI7RzgpErbXgy3f5zmvCMrtU39Kbl6sK6NeasZY95qVl+3q7E//3LPlNtffqyqvi7xEfdnm+y2S2P7u1NK2fBszZNS7jKzJpXWkzsf9Ay3jQJGJR3zCmx+CcDMSoHrw0+6e80Bjqli3wVVxWZm3VPs+0XS6qgU+0cB8at3cM5tp+xqU/Kk5JxzOc5LSs4552LBx75zzjkXHwblnpScc87FgeHVd84552LDOzo455yLEau1sVfqnicl55zLcV5955xzLhbMoDyLJvnzpOSccznOq++cc87FhlffOeeciwVDnpScc87FRxbV3nlScs65nGbZVX2XPV0ynHPObROrUEaf6khqJOkTSZ9J+kLS78Pt3SR9LGm6pGckNQi3NwzXp4f7u1Z3D09KzjmX48wy+2RgA3CMme1PMOnoCZIOAf4M3GlmPYBlwIXh8RcCy8Ltd4bHpVVl9Z2ke0hTFWlmV2b0IzgXAxWrV0cdQo3k7Zd6QsG4qpg4JeoQMqb8/KhDqJnS7Tu9Nse+MzMDEv+ZCsOPEcz7dm64/VHgd8D9wKBwGeDfBDNuK7xOSunalMZua+DOOediwoDMk1JrScnf/UPNbGjyAZLygXFAD+BeYAaw3MzKwkPmA4npmDsC8wDMrEzSCqAVUFJVAFUmJTN7tFIgxWa2NpOfyjnnXHzU4OXZEjPrm/5aVg70ktQCeBGo1WJ9tW1Kkg6V9CXwVbi+v6T7ajMI55xzdcgy/NTkkmbLgbeBQ4EWkhKFnE7AgnB5AdAZINzfHFiS7rqZdHS4CxiQuJCZfQYcVaPonXPORSSznncZ9r5rE5aQkFQEHAdMJkhOZ4SHnQ+8HC6/Eq4T7n8rXXsSZPiekpnNk7YIuDyT85xzzkWsdt9Tag88GrYr5QHPmtl/wtq0pyX9CRgPPBQe/xAwTNJ0YClwTnU3yCQpzZN0GGCSCoGrCDKjc865bFBLQzqY2edA7xTbZwIHpdi+HjizJvfIpPruYuAygl4UCwn6pl9Wk5s455yLkjL8RK/akpKZlQDn1UMszjnn6kIWDX6XSe+77pJelbRY0reSXpbUvT6Cc845t50MqFBmnxjIpPruSeBZggauDsBzwFN1GZRzzrnaU4vDDNW5TJJSsZkNM7Oy8PM40KiuA3POOVdL6uA9pbqSbuy7luHi/yRdDzxNEPbZwGv1EJtzzrnakEVTV6Tr6DCOIAklfpqfJe0z4Ia6Cso551ztUUxKQZlIN/Zdt/oMxDnnXB2IUdVcJjIa0UFST2BvktqSzOyxugrKOedcbYlPz7pMVJuUJP0W6EeQlF4DTgTeAzwpOedcNsiiklImve/OAPoDX5vZj4D9CUZ6dc45lw1yofddknVmViGpTFIz4FvCocidq019+63k4j8uJD/P+N9TLXl2SNuoQ6pSYcMKbn9+OoUNK8jPh9H/bc6w29tHHdYWGjfeyNVXj2GXriswgzvvPIhTT51Kp06rAGjSZCOrVzfg8ssGRBzpltp02Miv/j6XFq1LwcRrT7TipYfaRB1WSp26r+eGe2duWm/XZQPD7ujASw/F6N9uzSb5i1wmSWlsOFT5gwQ98lYDH2ZycUmnEkwCtZeZfSWpKzALuNLM7gmPGQKMNbN/hesFwCLgITO7Pulaowhe4F0fxvBjM5sSbr/GzMaGx3UF/mNmPZPOvYtgUMDOZlYRbmtLMIJtZ4IpfWeb2Unh+ZOB5Pmd70i0oUnaFxgWbu8CrAg/JcBPqjpX0mygbzhsUyKuBsBfgIEE/3S+BC4zs/nh/nJgIsHf02TgfDNbK2m1mTVJus4F4bUvT9o2AfjKzM5J2nYI8HegYfh5xsx+F57/VzbPgQJwrpl9ST3JyzMuu2UBN5zTnZJFhdzz2jQ+GtGcudPi+Upc6QZx7Vm7sn5tPvkFxh0vTmPM28346tPGUYe2ycUXj2fsuHbcfPPhFBSU07BhObfdetim/T/56XjWrmkQYYSplZeJob/vwPRJxRQ1LmfI8Kl8+m7TWP5bmD+zEZeduDcQ/Bt+/JPP+WB4i2iDSiGbet9VW31nZpea2XIze4Bg7ozzw2q8TAwmaH8anLTtW+Cq8As5leOAqcCZqjRfBnCeme1PMAf8XzMJQFIecBrBlLzfSdr1B+B1M9vfzPYGrk/aN8PMeiV9NrWfmdnExHaCuUJ+Fa4fW925KdwCNAX2MLPdgJeAF5J+7nXhNXoCGwkGx83kZ94LyAeOlJT8LfkocFEYe0+CkToSnqkUd70lJIA9eq9l4ewGfD23IWWleYx6uQWHDlhRnyHUkFi/Nh+AggIjv9Bi80Y8QHHxRnruu5gRw4MRwcrK8lmzRQIyjjpqHqNGdYkmwDSWflvI9EnFAKxbk8+8aQ1p3a404qiq1+vwVSya25BvFzSMOpStZVH1XZVJSVKfyh+gJVAQLqclqQlwBHAhW86hsRh4k80TP1U2mOC3+bkEMxqm8i7B/PCZ6Ad8AdzPlsmxPcFc8sCmIdnrjaRi4EfAz8PphTGzR4ANwDEpThlN5j/zYILS3EhgUNL2nQlKoZhZeX0nnnRatStl8cLNX5oliwpp3T7eX0R5ecZ9I7/imc8nMf7dpkwZH59SUrt2a1ixoiG/+OUnDBkygquu/oSGDcs27e/ZczHLljVi4cKmEUZZvbadNrBrz3V8Nb446lCq9Z3vLmXUyy2rPzACssw+cZCu+u72NPuM1F+cyQYBw81sqqQlkg5g8zS4fyYYKeLh5BMkNQKOJXhRtwXBl+sHKa59CkG1VsITktaFyw2AiqR9gwnG6nsZuEVSoZmVAvcCz0i6HHgDeMTMFobn7BpWfyVcYWajq/l5EzI9twcw18xWVto+FtiHIHEDm6o0TwSGh5uKKt2jJUGpLeFsghLnnsAVBOMXAtwJJKo8hwOPhvOdAJwt6YikaxxqZuuoRNJFwEUAjYj/F0VdqqgQlx6/J42blfHbh2azyx7rmDOlKOqwAMjPN3r0WMb99/VhypRW/OziTznr7MkMe2xfAPr1m8s7MSwlJWtUXM5vHpzNA7/tyNrV+VGHk1ZBYQWHHLecR/7cMepQUsuFNiUzO3o7r50o8UAwRNFgYEh47ZmSPgbOrXTOQOBtM1sn6XngN5KuTpQk2Jx8ZhN82SacV7lNKVxuAJwE/MLMVoX3HEDQ5jQiHO38BIIv/PHh+1gQVsFt48+9PedWlpx8RrN5Nsd1yfdItCmFy32BEjObK2kB8LCklma21Mz+IOkJ4HiCZz+YoCQJQfXdpjapqpjZUGAoQDO1rLXfrZZ8XUibDhs3rbduX0rJosLaunydWrOygM/eb8KB/VbFJimVlBRRUlLElCmtAHhvdGfOOjuYmzMvr4LDDp/PlVccH2WIaeUXGL95cDZvvbgT7/+vRdThVKtvv5VMn1TM8pIY/puNUdVcJjLpEl5j4bh5xwD/DBv4fwWcxZazSN0CXFdp22Dg2PCccUArtiyRnRe2d5xqZvMyCGUAQYlrYnjNI0iqwgu/qJ80sx8AY4Cj0vxMp0maEH76ZnDv6swAukiqXH9yAEF1I2xuU+plZleY2UaqNxjYM/x5ZwDNgNMTO81shpndT9DNf39Jrbb3B6kNUyYU07HbRtp23kBBYQX9Bi3no5HxffOgecsyGjcLqsMaNKqgz1GrmDcjPm0Jy5YVsXhxMR07BQXxXr2/Ye7cZgD07v0N8+c1o6QkriVd4xe3z2Xe9Ia8MHTnqIPJSL9B8a26A7KqTSmjER22wRnAMDPbNF6epHdI6koe9sb7kqAqbkzY3fxIgh5yG8JzfkTwJfv6NsYxGPiJmT0VXq8xMCtszzkE+CjszdYU2JWgHSslM3uRoCdhrTCzNZIeBe6QdLGZlUv6IVAMvLUt1ww7dZwF7JuoipR0NPAb4EFJJwOvmZkBuwHlwPLt/2m2X0W5uPfGjtzy5Ezy8mHk0y2ZMzV+va0SWrYt5Zq75pKXZ+TlwbuvtuDjN+KVRO+/rw/XXvsRhYUVLFrUhDvvCGar/k6/ubHs4JCwz4FrOPaMZcz8shH3jfwKgEdu68CYt5pFHFlqDYvK6XPkSu6+YZeoQ6lSXNqLMlFXSWkwQbtRsufZehDXm4Hx4fJpwFuJhBR6GfiLpBr/ChomnhNI6rEWJoL3CBJhF2CIpDKCEuM/zWxMWP1XuV3oYTO7O8Nbpzv3c0mJ9q5nCZ7H34Cp4favgNPCpLEtjgQWJLWNQdApZG9J7YEfAHdKWguUEZQ8y8POfpXblC41s1TteXVmzFvNYvvFU9msyUVcNmCPqMNIa+bMnbjqyq2r6O64/eAIosncF2OaMKBjr6jDyNiGdfmctX+vqMNIr6L6Q+JC1X3/hd2TzwO6h20SXYB2ZvZJfQTo4quZWtrB6h91GJnZ6u2CeMvbN94Jr7KKiVOqPygmlB/vThOVvV769Dgz2+Ymg0adOlunq36e0bEzrv3ldt2rNmTSpnQfQdfsRFvMKoKea84557KBKbNPDGRSfXewmfWRNB7AzJalefHVOedc3ORYm1KppHzCH0tSG7KqhtI553Zs2dTRIZPqu7sJep3tLOlmgmGDbqnTqJxzztWeXOoSbmZPSBpH8F6LgFPNbHKdR+acc277GSiL6rYymeSvC7AWeDV5m5lV+U6Pc865GIlJKSgTmbQp/ZfgRxLBdOjdCKZm2KcO43LOOVdLsqlNKZPqu32T18MRwi+ts4icc87tsGo8ooOZfSop3q+EO+ec2yyXSkqSfpG0mgf0ARZWcbhzzrk4ybWODgQzoyaUEbQxPV834TjnnKt1uVJSCl+abWpm19RTPM4552qRyJGODpIKzKxM0uH1GZBzzrlalgtJCfiEoP1ogqRXgOeANYmdZvZCHcfmnHNue1l2lZQyGWaoEbCEYAbYgQRzEQ2sy6Ccc87VoloaZkhSZ0lvS/pS0heSrgq3t5T0uqRp4Z87hdsl6W5J0yV9Hr5SlFa6ktLOYc+7SWx+eTb5R3TOOZcFarH3XRnwy/DVoKbAOEmvAxcAb5rZbZKuB64HrgNOJJjlejfgYOD+8M8qpUtK+UATtkxGCZ6UXFbJa1jjyYsjlU2T5gFMfSDSeeFqZK/rp0cdQs0srYVr1NI3tpktAhaFy6skTQY6AoOAfuFhjwKjCJLSIOCxcDbtjyS1kNQ+vE5K6ZLSIjP7w3b/FM4556JTsxHAW0sam7Q+1MyGpjpQUlegN/Ax0DYp0XwNtA2XOwLzkk6bH27bpqQUj2kInXPObZcadHQoyWQ6dElNCN5XvdrMVkqb04WZmbTtXSvSdXTov60Xdc45FyO1OJ+SpEKChPREUi/sbyS1D/e3B74Nty8AOied3incVqUqk5KZ1UZNpnPOuYjJMvtUe52gSPQQMNnM7kja9Qpwfrh8PvBy0vYfhr3wDgFWpGtPgm0YkNU551wWMaD2et8dDvwAmChpQrjt18BtwLOSLgTmAGeF+14DTgKmE8zL96PqbuBJyTnncpiovQ4CZvZemstt1eQT9rq7rCb38KTknHO5Lote4vGk5JxzOS6bhhnypOScc7nOk5JzzrlYyMFJ/pxzzmUzLyk555yLC29Tcs45Fx+elJxzzsWFl5Scc87FQ81GCY+cJyXnnMthwnvfOeecixMvKTlXc337reTiPy4kP8/431MteXZI2+pPqkc///NMDjp6GcuXFHLJifsBcOH1czm4/zLKSsWiOY2449rurFkVr/9WbTps5Fd/n0uL1qVg4rUnWvHSQ20ijalg6QbaPTKL/FWlAKw4sg3L+7ej/dDpFH6zHoD8deWUF+Uz9zc9KSjZQNffTWRj20YArO/ehG/P6xpJ7K3breeXt37FTq02YgbDn+vAy4932rT/tPPn8dNrZ3DO4YexcnmDSGKsTJY9WSle/3vqiKRyYCLBzzsZON/M1lbaPgv4gZktD8/ZB7iHYJbEPOAx4E/hBFYXAI8Ax5nZG+HxpwIvAmea2b8ljQLaA+vCMKYD44Ezw/V9w3sDPGxmd0u6K9zf2cwqwuteAPyVYA6SBsCdZvZguL2vmV2e9HOOAq4xs7Hheq/wniea2fCk424EzgXKCcYP/pmZfZwqZjM7I/Mnve3y8ozLblnADed0p2RRIfe8No2PRjRn7rRG9XH7jLz+79a88lhbrvnbjE3bxr/XjEf+2pmKcvHj6+Zy9qULefjPXSKMcmvlZWLo7zswfVIxRY3LGTJ8Kp++2zTSZ2v5YvGZndnQpTFaX84uN3/B2r2as+iiHpuOaf3cXCqK8jetl7ZpxNzf9Iwi3C2Ul4l//mVXZkxuSlFxGXc/N45PP9yJeTMa07rdevocvpRvFzaMOszNsqxNKd0kf7lknZn1MrOewEbg4hTblxKOZiupiGAekNvMbA9gf+Aw4NKka04EzklaHwx8Vum+54XX72VmZ5jZzYn1pHv3ChNSHnAawdTB36l0nWfCc/oBt0jKtAgxGHgv/JPwZzsUGAj0MbP9gGPZcrriLWLO8D7bbY/ea1k4uwFfz21IWWkeo15uwaEDVtTX7TMyaUwzVi3f8ve4T99rQUV5MGjyV+Ob0LrdxihCS2vpt4VMn1QMwLo1+cyb1pDW7Uojjam8eQM2dGkMgDXKZ2P7IgqWJz07M5qOW8qqA1tFFGHVlpU0ZMbkpgCsW1vA3JnFtN55AwAXXTedh2/flbgVTGprPqX6sKMkpWSjgR4ptn9IUCqCoBTxvpmNBDCztcDlwPWVrnOQpMJwauAewITtiKsf8AVwP0lJJJmZfQvMAHap7mLhZFxnAhcAx0lK/FrcnmDK4w3hNUvMbOF2xF0rWrUrZfHCzVUdJYsKad0+2i/Omjr+zMWMGdUi6jDSattpA7v2XMdX44ujDmWTgpINNJy7lvXdmmzaVjRtNeVNCyltu7k0V1iygS5/+oJOf/uKommrogh1Kzt3WMeue63mq8+bccjRJSz5piGzpjSp/sT6Voszz9a1HSopSSoATmRztVliez7BXCCvhJv2AcYlH2NmM4AmkpolNgFvAAOAQUnnJntC0oTw89dqwhsMPEVQBXhyOOVw5fi7A90JqgIBzk66/gSgb9LhhwGzwrhHASeH20cCnSVNlXSfpMqlsrQxS7pI0lhJY0vZUM2PtOM459IFlJeJt1+O32/2CY2Ky/nNg7N54LcdWbs6v/oT6oHWl9PhH9NZfFbnLarqmo5ZwqqDNj/L8uaFzLx1f+betA+Lz+xMu4dmkLeuPIqQN2lUXMaNd33B0Nt6UFEuzr5oDsOGdIs0pqqoIrNPHOwoSako/NIeC8wlmM43efvXQFvg9Rpe92mCKrxzCBJKZclVYb+q6iKSGhDMzviSma0EPiZIdglnh3E+RdD+k5iq/pmk6/cKf76EwWF8iTgHA5jZauAA4CJgMfBM2D6VUcxmNtTM+ppZ30Jqr958ydeFtOmwufqmdftSShZtlZdj6djTF3PQMcv5y893pfamU6td+QXGbx6czVsv7sT7/2sRdTiB8go6/GM6Kw9qxeo+LZO2G03GL2NV383brDCPiiZB1emGXRpT2qbRpg4RUcgvqODGu75g1H/b8sEbbWjfeR1tO67n3hfG8MjID2nddgN3/3scO7WOwS9uGVbdxaX6bofo6EDYflPVdknFwAiCNqW7gS+Bo5IPDEspq81sZVAzBmb2iaR9gbVmNjWxfRsMAFoQTDEMUEzQ2eA/4f5nkjs0VCcs+Z0ODAo7NQhoJampma0ys3KC0tMoSROB84F/bWvwtWHKhGI6dttI284bWPJ1If0GLee2y6qtpYzcAUct58yLFnLt4L3ZsD4epY+tGb+4fS7zpjfkhaE7Rx1MwIx2j81mY7silh/XbotdxZNXsrFdEWU7ba7OzV9VSnnjAsgThYvX0+Db9ZS2iaozgXH1H6Ywb2YxLz7aGYDZ05pw7lGHbzrikZEfctVZB8Sm911cquYysaMkpbTCnnhXAi9Jug94Avi1pGPN7I2w48PdwF9SnH49sL2/sg0GfmJmTwFIagzMCpPltugPfG5mm0pbkh4FTpP0MVBhZtPCXb2AOdsceS2pKBf33tiRW56cSV4+jHy6JXOmxqfnHcB1f5/OfgevpNlOZQx7/1OG/b0TZ1+8kMIGxs2PfQXAVxOaMOSmeFXh7HPgGo49Yxkzv2zEfSODOB+5rQNj3mpWzZl1p9GM1TT7aAkbOhbR5Y+TAFhyaifW7NuCpmOXsOrAllscXzRtFa1eWYDlCyS+ObcrFY2j+frau88K+g/6hllTGnPP82MAePSu7owdHc+qWxGfUlAmPCmFzGy8pM+BwWY2TNIg4B5J9wL5wDBgSIrz/pfmsk9ISnSvLjGzYysfECaeE9jcIxAzWyPpPeCUbfxxBhO0TSV7HriEoDPFPZJaAGUE7VMX1STmujLmrWaRflFW589Xbd0/ZuSzMSl5pPHFmCYM6Ngr6jC2sL5HU6b+48CU+765oPtW21b3abllFV+Evvy0BSft0y/tMT86/tD6CSZTcesOmMYOkZTMLGV3mMrbzeyUpOWJBD3iUp33L1JUd5nZBUnLKc+tfO+wZ99W/9vM7HtJq6nutVUMSff8UYrjX2FzZ4zDqogpbczOuSzkk/w555yLE09Kzjnn4iN7au88KTnnXK7zjg7OOefiwfCODs455+LDS0rOOediwSf5c845Fx9mXn3nnHMuPrz6zjnnXHx4UnLOORcXXlJyzjkXDwZUZE9W8qTknHM5znvfOeeci48s6n23o8w865xzO6zamnlW0sOSvpU0KWlbS0mvS5oW/rlTuF2S7pY0XdLnkvpkEqsnJeecy2VWg0/1/kUw/1uy64E3zWw34M1wHeBEYLfwcxFwfyY38Oo7t0OoWL+9kwPXr7zGjaMOoUZ2v3hs1CFk7LUF46MOoUby22/f+cHMs7VTfWdm70rqWmnzIDbPPfcoMAq4Ltz+mJkZ8JGkFpLam9midPfwpOScczlO5RknpdaSkn/DGGpmQ6s5p21SovkaaBsudwTmJR03P9zmSck553ZYmVfNAZSYWd9tvpWZSdv3VpS3KTnnXE6zzePfVffZNt9Iag8Q/vltuH0B0DnpuE7htrQ8KTnnXI6rrd53VXgFOD9cPh94OWn7D8NeeIcAK6prTwKvvnPOudxXSx0dJD1F0KmhtaT5wG+B24BnJV0IzAHOCg9/DTgJmA6sBX6UyT08KTnnXC6z2hvRwcwGV7Grf4pjDbispvfwpOScc7nOx75zzjkXF7X1nlJ98KTknHO5zpOSc865WDDARwl3zjkXB8K8+s4551yMVGRPUcmTknPO5TKvvnPOORcnXn3nnHMuPjwpOeeci4ftGmy13nlScs65XGZ4UnJuW/Ttt5KL/7iQ/Dzjf0+15Nkhbas/KUJxj/fnt07noKOXsnxJIZec3BuA6++aQqfu6wBo0rSc1avyufy7vSKMcmuFDSu4/fnpFDasID8fRv+3OcNu387pV2vBDw/am6Im5eTlQX6BMWT4VFYuy+eWi7vyzfwGtO20kRv/MZumLco3nTNlQhFXn7I7v75/NkcOXBFZ7DWY5C9yPnXFNpBkkh5PWi+QtFjSf8L1C8L1CUmf/ZOWl0qaFS6/EZ7TK7zuCZXuVR4eN0nSc5KKw+2rKx13gaQhlbZNkPR0pW2HSPo43DdZ0u/SxLx3LT62tPLyjMtuWcBN53Xjp/324OhBy+myW3ynMM+GeF9/oQ03/XjLv8Lbrt6Dy7/bi8u/24v3RrTkg5GtIoquaqUbxLVn7colx+3JJcfvQd9+q9izz5qowwLgL89N5/43pjBk+FQAnh2yM72PWMUj70+m9xGreGbIzpuOLS+Hh27uwAHfWRVVuJvV7XxKtcqT0rZZA/SUVBSuH8fWk1c9Y2a9kj6fJZYJ5hn5Vbh+bHj8YOC98M9k68LjegIbgYszCVDSXkA+cKSkxkm7HgUuCuPoCTybJuYvM7lXbdij91oWzm7A13MbUlaax6iXW3DogOh+s6xONsQ7aUxzVq2oqjLEOOqkJYx6tXW9xpQZsX5tPgAFBUZ+ocXl+3IrH45ozrFnLQXg2LOW8uHw5pv2vfxwG444aQUtWpdFFV7ACAZkzeQTA56Utt1rwMnh8mDgqW29kCQBZwIXAMdJalTFoaOBHhledjAwDBgJDEravjOwCMDMyusz8aTTql0pixc22LResqiQ1u1LI4wovWyLt7KeB65kWUkhC+cUVX9wBPLyjPtGfsUzn09i/LtNmTK+cfUn1TUZvx68K5cN2J3XHg9KmMtKCmnVNkg6LXcuY1lJIRD8e/jgf80ZeH5JZOFuVuczz9Yqb1Padk8D/xdW2e0HPAwcmbT/bElHJK0fambrqrjWYcAsM5shaRRBsns++QBJBcCJwPBwU5GkCUmHtCQogW26P0EJbk/gCuDJcPudwJTwPsOBR80sUe9UbcySLgIuAmhEcRU/jou7fgNLeOc/cSwlBSoqxKXH70njZmX89qHZ7LLHOuZMiTaB3vHSdFq3L2V5SQHXn7MrnXtsWV0rgcLpWx/4bUcuvHEheXH5tT8mCScTnpS2kZl9LqkrQYnktRSHPGNml2d4ucEESY7wzx+yOSklJ5/RwEPh8rqwCg4I2oSAvuFyX6DEzOZKWgA8LKmlmS01sz9IegI4Hjg3vHe/TGM2s6HAUIBmallr/9KXfF1Imw4bN623bl9KyaLC2rp8rcu2eJPl5RuHHb+UK0/bL+pQqrVmZQGfvd+EA/utijwpJUrCLVqXcfgJK/hqfDE7tS5lyTcFtGpbxpJvCmjRKig1Tf2siFsv6QrAiqX5fPJmU/Lz4bATI6rizaKkFJc8nq1eAf7G9lXd5QOnE5S6ZgP3ACdIahoesi6pjecKM9tY1bWSDAb2DK83A2gW3gMAM5thZvcTzBa5v6TIW7unTCimY7eNtO28gYLCCvoNWs5HI5tXf2JEsi3eZL0PW878mUWUfN0w6lBSat6yjMbNgi/3Bo0q6HPUKubNiDbW9WvzWLs6b9PyuHea0nXP9Rxy/EreeLYlAG8823JTu+JjH0/msU++5LFPvuTIgSu44tb50Sak8vLMPjHgJaXt8zCw3MwmSuq3jdfoD3xuZgMSGyQ9CpwGPFbTi0nKA84C9jWzheG2o4HfAA9KOhl4LZyqeDegHFi+jbHXmopyce+NHbnlyZnk5cPIp1syZ2pVTWvRy4Z4r7tzKvsdtIJmO5UxbPRYhv29MyP/3ZbvDCxhVIyr7lq2LeWau+aSl2fk5cG7r7bg4zeiTfjLFhfw+wu7AVBeBkeftpwDj17FHvuv5eaLuzL86Vbs3DHoEh5LWVRSkmVRsHEhabWZNam0rR9wjZkNDKvS/sqWPfIuNbMPwmP/BfzHzP4t6RHgYzN7IOla3wUuMbMTU90rVQxJ1XfPAX82s0OS9uWHsfQmaFPqA6wFyoAbzWxEdTGn0kwt7WD1r2q32w55jWPQsF8DFWvXRh1CxkYsGB91CDWS3376ODPru63nN2/Q1g5rV7lTb2rD5/19u+5VGzwpuW3mSanueFKqOztkUmp7TkbHDp9/d+RJyavvnHMu12VR4cOTknPO5bJER4cs4UnJOedynZeUnHPOxYYnJeecc/EQn3HtMuFJyTnncpmBWUXUUWTMk5JzzuU6Lyk555yLBe9955xzLla8o4Nzzrm4sApvU3LOORcL8ZnALxOelJxzLpclpkPPEp6UnHMu12VRl3Cf5M8553KYmWHl5Rl9MiHpBElTJE2XdH1tx+slJeecy3FWS9V34dxs9wLHAfOBMZJeMbMva+UGeEnJOedyn1Vk9qneQcB0M5tpZhuBp4FBtRmqT/LntpmkxcCcOrh0a6CkDq5bF7IpVsiueLMpVqi7eHcxszbberKk4QSxZaIRsD5pfaiZDU261hnACWb2k3D9B8DBZnb5tsZXmVffuW22Pf9R0pE0NurZLzOVTbFCdsWbTbFCfOM1sxOijqEmvPrOOedcphYAnZPWO4Xbao0nJeecc5kaA+wmqZukBsA5wCu1eQOvvnNxNLT6Q2Ijm2KF7Io3m2KF7Iu3xsysTNLlwAggH3jYzL6ozXt4RwfnnHOx4dV3zjnnYsOTknPOudjwpOSccy42PCk5lwFJxyQtd6u073v1H5FzucmTknOZ+VvS8vOV9t1Un4FkQtKeScsNK+07pP4jSk/Ss0nLf660b2T9R5Retj3fbOJdwl2kJL1KMONLSmb23XoMJx1VsZxqPQ6eBPqEyx8mLQPcV2k9DnZLWj4OuC5pvU5GDtlO2fZ8s4YnJRe1RAnke0A74PFwfTDwTSQRpWZVLKdaj4NsS6LpnqE/3x2IJyUXKTN7B0DS7ZXGDXtV0tiIwkqlu6RXCL5wEsuE692qPi0y2ZZEiyX1JmhSKAqXFX6KIo0stWx7vlnDk5KLi8aSupvZTNjUmaBxxDElSx6e/2+V9lVej4NOku4m+FJPLBOud4wurCp9DdyRYjmxHjfZ9nyzho/o4GJB0gkEw7TMJPiPvQvwMzMbEWlgSST1AnoAX5jZ5IjDSUvS+en2m9mj9RVLLvLnW3c8KbnYCHsxJXo1fWVmG6KMJ5mk/wO+D4wDDgZuNbMHo42qZiTtBCy3GP6nl3QgMM/Mvg7XfwicTjBf1+/MbGmU8WUizs83m3iXcBcLkoqBXwGXm9lnQBdJAyMOK9nZQC8zGwwcCFwUcTxpSfq/RLdlSQ0lvQXMAL6RdGy00aX0D2AjgKSjgNuAx4AVxHCg0yx8vlnDk5KLi0cIvpQODdcXAH+KLpytbDCztQBmtoT4/985G5gSLp9PUCXaBvgOcEtUQaWRn1QaOptgxtPnzew3BFWmcZNtzzdreEcHFxe7mtnZkgYDmNlaSXHqWlu5x92uSetxep8qYWNSNdIA4GkzKwcmS4rj//t8SQVmVgb0Z8uSaBzjzbbnmzX84bm42CipiLA7raRdgdi0KbFl7zuIZ4+7ZBsk9SR41+to4JqkfcXRhJTWU8A7kkqAdcBoAEk9CKrw4ibbnm/W8KTk4uK3wHCgs6QngMOBCyKNKEnifarKJHUmmH0z5f4IXQX8m6BK6U4zmwUg6SRgfJSBpWJmN0t6E2gPjEwqheQBl0cXWZWy6vlmE+9952JDUivgEILqsY/MrCTikFKS1AY4k2DUiQ7Ai2Z2Tfqz4kNSWzOL02gZKUlqTDDSxzlmdnLU8WQqW55vXHlJycVJI2AZwb/LvSVhZu9GHBMAkpoSfEGeC+wOvAB0M7NOkQaWIUktCLpYnwvsRZBMY0dSA+BkgjgHEAx++0CkQWUgW55vNvCSkouFcGTos4EvgIpws8WlA4GkdcAnBCOCv2dmJmmmmXWPOLQqhW10gwi+KHsDTYFTgXfNrCLNqfVO0vEEJc/jgbeBZ4B7zKxrlHGlk03PN5t4UnKxIGkKsF+cXphNJulqgrajxgSN8s8Ar8c1KUl6EjgSGAk8DbwFTDezOI7Th6QKgs4NFyS1z8Q26Wfb880mcX/Xwu04ZgKFUQdRFTO7y8wOYXMvvJeADpKuk7R7dJFVaW+CqtDJwOSwu3KcfwPtQzAFxBuSXpd0IZAfcUzpZNvzzRpeUnKxIOl5YH/gTZK6gpvZlZEFVY2wS/C5wFlmFrsXPMMRBwYTVIuWAHsAPePeCC/pMIK4Twc+I+hIEsdRHbLy+cadJyUXC1UNcOkDW9YOSQcQJNAzgflmdljEIVVLUh5wLEHvux9HHU862fh848qTknMZkLSK1NUzIuiQ0ayeQ9om4SgZR8alV2OCpLQztZrZp/UVy/aI6/PNJt4l3EVK0rNmdpakiaT40jez/SIIaytm1jSxLGm8mfWOMp7qSLqH9G0ccfvSvD1p+QCC0dgTDDimfsNJLwufb9bwpOSidnX4Z5xGBK9ONlQvJM/a+3uCETNiy8yOTiyHSf/odMfHQFY932zi1XcuUpI+NbM+koaZ2Q+ijicTiZijjiNT2VCyS+bPd8fmJSUXtQaSzgUOk/S9yjvN7IUIYtpKpdhaVI41LnFWwX/zrFv+fGuRJyUXtYuB84AWwCmV9hnBcD5xkBzbO5XW4xRnVkpqoxHQSdLdyfvj/GqAq11efediQdKFZvZQ1HHkiqTeggKKgLWJXcSwt2BVrwQkxO3VgGx7vtnEk5KLhXBE6J8DXczsIkm7AXuY2X8iDg0ASacAn5vZnHD9/whe7pwDXJUYGsdtu3D09V0IhutZHnE4LiI+zJCLi4cJpkNPvHQYt+nQbwYWA0gaCHwf+DHwCjEcxVpSI0lXSxoi6aK4z4Yq6ScEg/HeA3wlKRYD8VYl255vNvGk5OJiVzP7C1AKwXToBFUhcWFhTBBMYfGQmY0zs38STPQWN48CfYGJwEls+R5QHF0N7GNmhxL8YnJDtOFUK9ueb9bw7O7iIu7ToUtSE4K2g/7AfUn7GkUTUlp7m9m+AJIeIph2I842mtliADObKalh1AFVI9ueb9bwpOTiItbToQN3AROAlQSjQo8FkNQbWBRdWFUqTSyYWVkw+k2sVe5xt8V6DHvfZdvzzRre0cHFRtynQ5fUEdgZ+CwxiZuk9kChmc2NNLhKJJUDa9hcBZroIRbL3mFZ2Psu8Xxhyx54sXy+2cSTkotUrgzE6eqGpEbAKWb2XNSxuPrhSclFStLb4WIjgobjzwh+29wPGBs2fLttJOloYJ9wdZKZjYownIxIygcGsHl69NFmdka0UVUvfK3hNGCwmZ0cdTzZytuUXKQSA29KegHoY2YTw/WewO8iDC2rhVWNLwDr2Tzi9plhZ5LTzGxBZMFVQdJ3COYkOomg48DhQLekXo+xI6kBcDJB3AOA54nhKwLZxEtKLhYkfWFm+1S3LS7CL/3EdN0Lzawsyngqk/Qi8LKZ/avS9h8Cp5vZoJQnRkTSfGAucD/wkpmtkjTLzLpFHFpKko5nc0nubeAZ4B4z6xplXLnAk5KLBUlPETQcPx5uOg9oYmaDo4tqM0k3EHRo+EO4PhdYDjQAHjWzWyMMbyuSppjZHjXdFxVJdwGnApOAJ4GXgYlm1j3CsKokqQIYDVyQGM1D0sy4xptN/OVZFxc/Inij/6rw82W4LS7OZMsXJJeEExDuQ1B9Ezcp/2+HU4znp9oXJTO7GuhG8Iz7AVOANpLOCt8Pi5s+wIfAG5Jel3QhMXyu2chLSs5loPIcP5IuSFSNSRpnZgdEFlwKku4EmgBXm9macFtj4E5gfdze+5F0uZkNSVovZHNnhwFm1jqy4Koh6TCCOE8n6KjzopkNjTaq7OVJycWCpMMJOjbsQlIHnLhUh0iaSjAMTmml7Q0JerXtFk1kqYVf6rcSvIA8J9zchWB4nBsq/xxRSzexn6QiM1tX3zGlI2mkmR1faVsecCxwjpn9OJrIsp8nJRcLkr4iGCV8HFCe2G5mSyILKomkW4B2wOWJ3mBhyWMI8LWZxXKstrC3XY9wdYaZrZX0NzO7Jsq4KvPZZl2CJyUXC5I+NrODo46jKuG7MzcDPyEoeQjoDDwE3BS33nfpSJprZl2ijiOZpDI2z0m0xS5iOEKCpJlAlYk95jMRx5onJRcLkm4jaCh+gaSBWOM2okOlksf0uFUrZULSPDPrHHUcybKt5CFpCUEPwVSD3plX3207f3nWxUWilJToMCCCEcOPiSacLUk6KsXmAxMDcZrZu/UbUXqSWla1i3hNCZKt5njiqRuelFxcjEqxLU7F+F+l2GYEwyF1Jn7dgcexebruymLVySGUbWPbeWKvI56UXFysTlpuBAwEJkcUy1bM7JTk9bC34E3A18AVkQSVRlxHQkijV2JB0p/N7Lqk9a16usXAplHNJTU0sw1J64eY2UfRhJX9vE3JxVLY1XqEmfWLOpZkkvoDvyEohdxiZq9HHFJKkr5vZo+Hy4eb2ftJ+7Z4JygOknvfpXgnLHbtTdXEm1U9CePGR3RwcVUMdIo6iARJJ0v6gKDH1U1mdnRcE1LoF0nL91Tal21tIXH8zVlVLKdadzXg1XcuFiRNZPOXTz7QBvhDdBFt5VVgPrAEuFbStck7zey7kURVtWz70iwOZ/HNA4rC5USnjKJII0vNqlhOte5qwJOSi4uBSctlwDcxe/fn6KgDqKFs+9L8GrgjxXJiPW4S07WLLaduF9AxurCyn7cpOZeDJK0FphN8Se4aLhOudzezxlHFlguybfr2bOJJybkMVKpe3Eo4YnhsSNol3X4zm5Nuf32T9L10+32EhB2HJyXnMpCFX/Jx7EZdpXB+ognhB7Zs94rdCAmSXkm3P4ZtjFnD25Scy0BVSSccGXowm0fijos2UQdQQ98DziF4Gfll4Ckzm57+lEgdCswDngI+Jp6dR7KSl5Scy4CkZsBlBI3YrwCvA5cDvwQ+i+H04lk5YGg48vog4GygFXCjmb0TbVRbCwfoPY7gF5L9gP8SJNIvIg0sB3hJybnMDAOWEcw2+hPg1wS/HZ9qZhMijKsqzQl6NKYcMJRg4Ns4Wg+sAFYSzK3VKNpwUjOzcmA4MDx80XswMErS7+P2YnK28ZKScxmQNNHM9g2X84FFQBczWx9tZKll26gCko4hqL47CHgDeNrMxkYbVXphMjqZICF1JShBP2xmC6KMK9t5UnIuA9k2lEyqoXkk9YnbVCAJYUeHz4H3CEpyW3wxxXD69seAnsBrBAl0UsQh5QxPSs5lQFI5sCaxSjDKwFriOwldz8pflHFOpNn23k+YRBP/HpK/RGP57yGbeFJybgcRx4FNU5HUBMDMVld3rMs9PiCrczuO30cdQDqSLpE0l6B7/RxJcyRdGnVcrn55UnIuB0n6ftLy4QBm9lK4fnlEYVVJ0k3AKUA/M2tlZq0Ixhs8MdzndhBefedcDsq2+X4kTQH2r9ybUVIRwXtgu0cTmatvXlJyLjdl29QVlqp7vZmtAyoiiMdFxJOSc7kp26auWBDO6ruF8P2lRRHE4yLi1XfO5aBsm7pC0j4EY969B4wLN/cFDgcG+fA9Ow5PSs7loGwb1RxAUiPgXGCfcNOXwBNxHTXD1Q1PSs7tACS1Ao4C5prZuOqOj4vEKOxm9kTUsbj64W1KzuUgSf+R1DNcbg9MAn4MDJN0dZSxpSKpmaQbJA2RdJwClwMzgbOijs/VHy8pOZeDJH1hZvuEy78G9jSzH0pqCrwfw5lyX2bzKOz9gZ0J2r+uiuko7K6O+NQVzuWm0qTl/sCDAGa2Khy3LW66J43C/k9iPgq7qzuelJzLTfMkXQHMB/oQzP2TeBm1MMrAqrApiZpZuaT5npB2TF5951wOkrQz8AegPXCvmY0Mtx8NHGBmf4syvsqSRmFPvNgb61HYXd3xpOTcDiTsdn2KmT0XdSzOpeK975zLcZLyJZ0kaRjBCNxnRx1TZZIaSbo67H13kSRvWthBeUnJuRwl6TsEL6OeBHxCMDpCdzNbG2lgKUh6hqBdaTRwIjDHzK6KNioXBU9KzuUgSfOBucD9wEthr7tZZtYt4tBSkjQxqfddAfBJ3EYyd/XDq++cy03/BjoQVNWdIqkx8RyINSG5911ZlIG4aHlJybkcJUlAP2AwQRVec+BC4LW4TTWe1PsOgh533vtuB+VJybkdgKRC4ATgHGCAmbWOOCTnUvKk5NwORlJROHmec7Hj3S6dy0GSJpK+DSlWY985l+AlJedyUNJ8SgL+S9CmtEkc51NyDryk5FxOSk46kjZ4EnLZwruEO+eciw0vKTmXgyQlv3haJKk3mwc7xcw+rf+onKuetyk5l4MkvZ1mt5nZMfUWjHM14EnJOedcbHibknM5SNJukl6SNEnSU5I6Rh2Tc5nwpORcbnqYoCv46cCnwD3RhuNcZrz6zrkcJGmCmfVKWv/UR9122cB73zmXmxpV6nG3RQ88733n4spLSs7lIO9957KVJyXndjCSCs2stPojnat/3tHBuR2AAv0lPQTMjzoe56riScm5HCbpEEl3A3OAl4F3gT2jjcq5qnn1nXM5SNItwJnAXOAp4EVgrJl1izQw56rhve+cy00/AaYC9wOvmtkGSf4bqIs9r75zLje1B/4EnALMkDSMoFu4/yLqYs2r75zLcZIaAgOBwcCRwJtmdm60UTmXmicl53YgkpoCp5nZY1HH4lwqXpR3LgdJ+mHUMTi3Lbyk5FwOklTVAKzfBTqamf9C6mLJk5JzOU6SgPOA64AvgZvN7PNoo3IuNf9tybkcFfa0uwC4BvgIOMPMpkQalHPV8KTkXA6SdBlwFfAmcIKZzY42Iucy49V3zuUgSRXAt8BiIPk/uYAKM9s/ksCcq4aXlJzLTamGExLQGbihnmNxLmOelJzLQWY2J7EcTu53LsFYeLOA56OKy7nqeFJyLgdJ2p1gBIfBQAnwDEF1/dGRBuZcNbxNybkcFLYpjQYuNLPp4baZZtY92sicS88HZHUuN30PWAS8LelBSf0J2pScizUvKTmXwyQ1BgYRVOMdAzwGvGhmIyMNzLkqeFJybgchaSeCzg5nm1n/qONxLhVPSs4552LD25Scc87Fhicl55xzseFJybk6JKlc0gRJkyQ9J6l4O671L0lnhMv/lLR3mmP7STpsG+4xW1LrTLdXOmZ1De/1O0nX1DRGl9s8KTlXt9aZWS8z6wlsBC5O3hmO5F1jZvYTM/syzSH9gBonJeei5knJufozGugRlmJGS3oF+FJSvqS/Shoj6XNJP4NgHiRJQyRNkfQGsHPiQpJGSeobLp8g6VNJn0l6U1JXguT387CUdqSkNpKeD+8xRtLh4bmtJI2U9IWkf5LBu0ySXpI0Ljznokr77gy3vympTbhtV0nDw3NGS9qzVp6my0k+zJBz9SAsEZ0IDA839QF6mtms8It9hZkdKKkh8L6kkUBvYA9gb6AtwQR9D1e6bhvgQeCo8FotzWyppAeA1Wb2t/C4J4E7zew9SV2AEcBewG+B98zsD5JOBi7M4Mf5cXiPImCMpOfNbAnQGBhrZj+X9H/htS8HhgIXm9k0SQcD9xG8M+XcVjwpOVe3iiRNCJdHAw8RVKt9Ymazwu3HA/sl2ouA5sBuwFHAU2ZWDiyU9FaK6x8CvJu4lpktrSKOY4G9g0loAWgmqUl4j++F5/5X0rIMfqYrJZ0WLncOY10CVBCMsQfwOPBCeI/DgOeS7t0wg3u4HZQnJefq1joz65W8IfxyXpO8CbjCzEZUOu6kWowjDzjEzNaniCVjkvoRJLhDzWytpFFAoyoOt/C+yys/A+eq4m1KzkVvBHCJpEIIRvgOhwd6Fzg7bHNqD6Qa4fsj4ChJ3cJzW4bbVwFNk44bCVyRWJHUK1x8l2BaCySdCOxUTazNgWVhQtqToKSWkAckSnvnElQLrgRmSTozvIck+QSDrkqelJyL3j8J2os+lTQJ+AdBLcaLwLRw32PAh5VPNLPFwEUEVWWfsbn67FXgtERHB+BKoG/YkeJLNvcC/D1BUvuCoBpvbjWxDgcKJE0GbiNIiglrgIPCn+EY4A/h9vOAC8P4viAYi8+5lHyYIeecc7HhJSXnnHOx4UnJOedcbHhScs45FxuelJxzzsWGJyXnnHOx4UnJOedcbHhScs45Fxv/D6SrBscv14jhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ConfusionMatrixDisplay(confusion_matrix=cm2, display_labels=[x for x in categories_l2.values()]).plot(xticks_rotation='vertical')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3077796",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a010e4e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save({'cm1':cm1,'cm2':cm2,\n",
    "                              'all_gt_scores': all_gt_scores,\n",
    "                              'all_gt_scores_l2': all_gt_scores_l2,\n",
    "                              'all_predictions':all_predictions,\n",
    "                              'all_predictions_l2': all_predictions_l2, 'metrics': metrics}, 'run%d_hfcos.pth' % run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "82caac67",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9ad1346",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "97825ee788e89970c5e1326e3a8e2ffc83913c5a8306b832b194d1fccd0a0039"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}