coco-pano-multimages.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# COCO Panoptic scoring for multiple images\n",
    "\n",
    "TL;DR: we need to delay indicators computation until we gather all matching scores.\n",
    "\n",
    "This notebook contains 2 parts:\n",
    "\n",
    "1. simple walkthrough on edge cases, showing the labeling and the matching steps work on empty predictions and/or targets\n",
    "2. a summary with reusable functions\n",
    "\n",
    "We assume detections are rasterized as binary maps with disjoint shapes here, but providing label maps directly should work just as well, as long as all elements belong to the same class.\n",
    "\n",
    "Missing here:\n",
    "- background management: we just discard it here\n",
    "- multiclass (seems to be only an extra for loop to add)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Global init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "W_TtoP_global = []\n",
    "W_PtoT_global = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inner loop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample detection 1: P and T both not empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "aa = np.zeros((10, 10), dtype=bool)\n",
    "aa[1:5,1:5] = True\n",
    "bb = np.zeros((10, 10), dtype=bool)\n",
    "bb[1:5,2:6] = True\n",
    "bb[6:9,6:9] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 1, 1, 1, 1, 0, 0, 0, 0, 0],\n",
       "       [0, 1, 1, 1, 1, 0, 0, 0, 0, 0],\n",
       "       [0, 1, 1, 1, 1, 0, 0, 0, 0, 0],\n",
       "       [0, 1, 1, 1, 1, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aa*1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 1, 1, 1, 1, 0, 0, 0, 0],\n",
       "       [0, 0, 1, 1, 1, 1, 0, 0, 0, 0],\n",
       "       [0, 0, 1, 1, 1, 1, 0, 0, 0, 0],\n",
       "       [0, 0, 1, 1, 1, 1, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 1, 1, 1, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 1, 1, 1, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 1, 1, 1, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bb*1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from coco_pano_ext_demo.coco import _compute_labelmap, _compute_iou"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "laa, lbb = _compute_labelmap(aa), _compute_labelmap(bb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 1, 1, 1, 1, 0, 0, 0, 0, 0],\n",
       "       [0, 1, 1, 1, 1, 0, 0, 0, 0, 0],\n",
       "       [0, 1, 1, 1, 1, 0, 0, 0, 0, 0],\n",
       "       [0, 1, 1, 1, 1, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "laa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 1, 1, 1, 1, 0, 0, 0, 0],\n",
       "       [0, 0, 1, 1, 1, 1, 0, 0, 0, 0],\n",
       "       [0, 0, 1, 1, 1, 1, 0, 0, 0, 0],\n",
       "       [0, 0, 1, 1, 1, 1, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 2, 2, 2, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 2, 2, 2, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 2, 2, 2, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lbb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.80681818, 0.6       ]), array([0.80681818, 0.6       , 0.10714286]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wtp, wpt = _compute_iou(laa, lbb)\n",
    "wtp, wpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.6]), array([0.6       , 0.10714286]))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove background components\n",
    "wtp, wpt = wtp[1:], wpt[1:]\n",
    "wtp, wpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0.6], [0.6, 0.10714285714285714])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update global matching lists\n",
    "W_TtoP_global.extend(wtp.tolist())\n",
    "W_PtoT_global.extend(wpt.tolist())\n",
    "W_TtoP_global, W_PtoT_global"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample detection 2: P empty, T not empty"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(we reuse `laa` as fake target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cc = np.zeros_like(aa)\n",
    "lcc = _compute_labelmap(cc)\n",
    "lcc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.84, 0.16]), array([0.84]))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wtp, wpt = _compute_iou(laa, lcc)\n",
    "wtp, wpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.16]), array([], dtype=float64))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove background components\n",
    "wtp, wpt = wtp[1:], wpt[1:]\n",
    "wtp, wpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0.6, 0.16], [0.6, 0.10714285714285714])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update global matching lists\n",
    "W_TtoP_global.extend(wtp.tolist())\n",
    "W_PtoT_global.extend(wpt.tolist())\n",
    "W_TtoP_global, W_PtoT_global"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample detection 3: P not empty, T empty"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(we reuse `lbb` as fake preduction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd = np.zeros_like(aa)\n",
    "ldd = _compute_labelmap(dd)\n",
    "ldd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.75]), array([0.75, 0.16, 0.09]))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wtp, wpt = _compute_iou(ldd, lbb)\n",
    "wtp, wpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([], dtype=float64), array([0.16, 0.09]))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove background components\n",
    "wtp, wpt = wtp[1:], wpt[1:]\n",
    "wtp, wpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0.6, 0.16], [0.6, 0.10714285714285714, 0.16, 0.09])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update global matching lists\n",
    "W_TtoP_global.extend(wtp.tolist())\n",
    "W_PtoT_global.extend(wpt.tolist())\n",
    "W_TtoP_global, W_PtoT_global"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample detection 4: both P and T empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1.]), array([1.]))"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wtp, wpt = _compute_iou(ldd, ldd)\n",
    "wtp, wpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([], dtype=float64), array([], dtype=float64))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove background components\n",
    "wtp, wpt = wtp[1:], wpt[1:]\n",
    "wtp, wpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0.6, 0.16], [0.6, 0.10714285714285714, 0.16, 0.09])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Update global matching lists\n",
    "W_TtoP_global.extend(wtp.tolist())\n",
    "W_PtoT_global.extend(wpt.tolist())\n",
    "W_TtoP_global, W_PtoT_global"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from coco_pano_ext_demo.iou import compute_matching_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.19999999999999998, 0.3333333333333333, 0.6)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# final agregation\n",
    "pairing_threshold = 0.5\n",
    "df = compute_matching_scores(np.array(W_TtoP_global), np.array(W_PtoT_global), pairing_threshold)\n",
    "COCO_SQ = df[\"IoU\"].mean() if len(df) > 0 else 0\n",
    "COCO_RQ = df[\"F-score\"].iloc[0] if len(df) > 0 else 0\n",
    "COCO_PQ = COCO_SQ * COCO_RQ\n",
    "\n",
    "COCO_PQ, COCO_RQ, COCO_SQ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IoU</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F-score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.6</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   IoU  Precision  Recall   F-score\n",
       "0  0.6       0.25     0.5  0.333333"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_matching_weights(target_binary_image, pred_binary_image) -> tuple[list[float], list[float]]:\n",
    "    # extract connected components\n",
    "    T = _compute_labelmap(target_binary_image)\n",
    "    P = _compute_labelmap(pred_binary_image)\n",
    "    # computes IoUs\n",
    "    wtp, wpt = _compute_iou(T, P)\n",
    "    # remove background components\n",
    "    wtp, wpt = wtp[1:], wpt[1:]\n",
    "    return wtp, wpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Iterable\n",
    "import pandas as pd\n",
    "def compute_pq_score_list_single_class(targets: Iterable[np.ndarray], predictions: Iterable[np.ndarray]) -> tuple[float, float, float, pd.DataFrame]:\n",
    "    # Init global accumulators\n",
    "    W_TtoP_global = []\n",
    "    W_PtoT_global = []\n",
    "    # loop over predictions (single class, TODO repeat for each class)\n",
    "    for T0, P0 in zip(targets, predictions):\n",
    "        # Compute pairwise matching scores, exluding background\n",
    "        wtp, wpt = compute_matching_weights(T0, P0)\n",
    "        # Update global matching lists\n",
    "        W_TtoP_global.extend(wtp.tolist())\n",
    "        W_PtoT_global.extend(wpt.tolist())\n",
    "    # report final score\n",
    "    pairing_threshold = 0.5\n",
    "    df = compute_matching_scores(np.array(W_TtoP_global), np.array(W_PtoT_global), pairing_threshold)\n",
    "    COCO_SQ = df[\"IoU\"].mean() if len(df) > 0 else 0\n",
    "    COCO_RQ = df[\"F-score\"].iloc[0] if len(df) > 0 else 0\n",
    "    COCO_PQ = COCO_SQ * COCO_RQ\n",
    "    \n",
    "    return COCO_PQ, COCO_RQ, COCO_SQ, df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "COCO_PQ: 45.7% = COCO_RQ: 57.1% × COCO_SQ: 80.0%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IoU</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F-score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.6</td>\n",
       "      <td>0.4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.571429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.285714</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   IoU  Precision  Recall   F-score\n",
       "0  0.6        0.4     1.0  0.571429\n",
       "1  1.0        0.2     0.5  0.285714"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "COCO_PQ, COCO_RQ, COCO_SQ, df = compute_pq_score_list_single_class([aa, aa, cc, dd], [aa, bb, bb, dd])\n",
    "print(f\"COCO_PQ: {COCO_PQ*100:.1f}% = COCO_RQ: {COCO_RQ*100:.1f}% × COCO_SQ: {COCO_SQ*100:.1f}%\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}