From 354abeb6f9223028c92921580c1f375c599b024e Mon Sep 17 00:00:00 2001 From: Federico Berto Date: Wed, 2 Oct 2024 23:37:51 +0900 Subject: [PATCH] [Notebook] CVRPLIB eval --- examples/2.eval-cvrplib.ipynb | 1063 +++++++++++++++++++++++++++++++++ 1 file changed, 1063 insertions(+) create mode 100644 examples/2.eval-cvrplib.ipynb diff --git a/examples/2.eval-cvrplib.ipynb b/examples/2.eval-cvrplib.ipynb new file mode 100644 index 0000000..328a193 --- /dev/null +++ b/examples/2.eval-cvrplib.ipynb @@ -0,0 +1,1063 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Evaluate on CVRPLib" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading http://vrp.galgos.inf.puc-rio.br/media/com_vrp/instances/Vrp-Set-A.tgz\n", + "Downloading http://vrp.galgos.inf.puc-rio.br/media/com_vrp/instances/Vrp-Set-B.tgz\n", + "Downloading http://vrp.galgos.inf.puc-rio.br/media/com_vrp/instances/Vrp-Set-E.tgz\n", + "Downloading http://vrp.galgos.inf.puc-rio.br/media/com_vrp/instances/Vrp-Set-F.tgz\n", + "Downloading http://vrp.galgos.inf.puc-rio.br/media/com_vrp/instances/Vrp-Set-M.tgz\n", + "Downloading http://vrp.galgos.inf.puc-rio.br/media/com_vrp/instances/Vrp-Set-P.tgz\n", + "Downloading http://vrp.galgos.inf.puc-rio.br/media/com_vrp/instances/Vrp-Set-X.tgz\n" + ] + } + ], + "source": [ + "# If you did not download the VRPLIB, you may run the following\n", + "from routefinder.data.download_vrplib import download_vrplib\n", + "\n", + "download_vrplib() " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/botu/mambaforge/envs/routefinder/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "/home/botu/mambaforge/envs/routefinder/lib/python3.12/site-packages/lightning/pytorch/utilities/parsing.py:208: Attribute 'env' is an instance of `nn.Module` and is already saved during checkpointing. It is recommended to ignore them using `self.save_hyperparameters(ignore=['env'])`.\n", + "/home/botu/mambaforge/envs/routefinder/lib/python3.12/site-packages/lightning/pytorch/utilities/parsing.py:208: Attribute 'policy' is an instance of `nn.Module` and is already saved during checkpointing. It is recommended to ignore them using `self.save_hyperparameters(ignore=['policy'])`.\n", + "Provided file name ['data/cvrp/val/100.npz', 'data/ovrp/val/100.npz', 'data/ovrpb/val/100.npz', 'data/ovrpbl/val/100.npz', 'data/ovrpbltw/val/100.npz', 'data/ovrpbtw/val/100.npz', 'data/ovrpl/val/100.npz', 'data/ovrpltw/val/100.npz', 'data/ovrptw/val/100.npz', 'data/vrpb/val/100.npz', 'data/vrpl/val/100.npz', 'data/vrpbltw/val/100.npz', 'data/vrpbtw/val/100.npz', 'data/vrpbl/val/100.npz', 'data/vrpltw/val/100.npz', 'data/vrptw/val/100.npz', 'data/cvrp/val/50.npz', 'data/vrptw/val/50.npz'] not found. Make sure to provide a file in the right path first or unset val_file to generate data automatically instead\n", + "Provided file name ['data/cvrp/test/100.npz', 'data/ovrp/test/100.npz', 'data/ovrpb/test/100.npz', 'data/ovrpbl/test/100.npz', 'data/ovrpbltw/test/100.npz', 'data/ovrpbtw/test/100.npz', 'data/ovrpl/test/100.npz', 'data/ovrpltw/test/100.npz', 'data/ovrptw/test/100.npz', 'data/vrpb/test/100.npz', 'data/vrpl/test/100.npz', 'data/vrpbltw/test/100.npz', 'data/vrpbtw/test/100.npz', 'data/vrpbl/test/100.npz', 'data/vrpltw/test/100.npz', 'data/vrptw/test/100.npz', 'data/cvrp/test/50.npz', 'data/vrptw/test/50.npz'] not found. Make sure to provide a file in the right path first or unset test_file to generate data automatically instead\n", + "/home/botu/mambaforge/envs/routefinder/lib/python3.12/site-packages/rl4co/models/rl/reinforce/reinforce.py:204: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n", + " state_dict = torch.load(checkpoint_path, map_location=map_location)[\"state_dict\"]\n" + ] + } + ], + "source": [ + "import torch\n", + "\n", + "from routefinder.models.model import RouteFinderBase, RouteFinderMoE\n", + "from routefinder.models.baselines.mvmoe.model import MVMoE\n", + "from routefinder.models.baselines.mtpomo.model import MTPOMO\n", + "from routefinder.envs.mtvrp import MTVRPEnv, MTVRPGenerator\n", + "\n", + "\n", + "# Choose your model\n", + "\n", + "PATH = \"../checkpoints/100/rf-transformer.ckpt\"\n", + "model = RouteFinderBase.load_from_checkpoint(PATH, map_location=\"cpu\")\n", + "\n", + "# PATH = \"../checkpoints/100/mtpomo.ckpt\"\n", + "# model = MTPOMO.load_from_checkpoint(PATH, map_location=\"cpu\")\n", + "\n", + "# PATH = \"../checkpoints/100/mvmoe.ckpt\"\n", + "# model = MVMoE.load_from_checkpoint(PATH, map_location=\"cpu\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "policy = model.policy\n", + "\n", + "# Create env\n", + "generator = MTVRPGenerator(num_loc=100, variant_preset=\"all\")\n", + "env = MTVRPEnv(generator, check_solution=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average reward: -19.634\n" + ] + } + ], + "source": [ + "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", + "policy = policy.to(device).eval()\n", + "\n", + "td_test = env.reset(env.generator(32))\n", + "\n", + "# Test the model\n", + "with torch.amp.autocast(\"cuda\"):\n", + " with torch.inference_mode():\n", + " out = policy(td_test.clone().to(device), env, phase=\"test\", decode_type=\"greedy\", return_actions=True)\n", + " actions = out['actions'].cpu().detach()\n", + " rewards = out['reward'].cpu().detach()\n", + "\n", + "print(f\"Average reward: {rewards.mean().item():.3f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### VRPLib Evaluation\n", + "\n", + "Note: run \n", + "\n", + "```\n", + "python routefinder/data/download_vrplib.py\n", + "```\n", + " to download the VRPLib instances (then move them under this dir)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found VRP file: A-n37-k6.vrp\n", + "Found VRP file: A-n33-k5.vrp\n", + "Found VRP file: A-n44-k6.vrp\n", + "Found VRP file: A-n48-k7.vrp\n", + "Found VRP file: A-n39-k5.vrp\n", + "Found VRP file: A-n63-k10.vrp\n", + "Found VRP file: A-n36-k5.vrp\n", + "Found VRP file: A-n34-k5.vrp\n", + "Found VRP file: A-n55-k9.vrp\n", + "Found VRP file: A-n69-k9.vrp\n", + "Found VRP file: A-n61-k9.vrp\n", + "Found VRP file: A-n38-k5.vrp\n", + "Found VRP file: A-n53-k7.vrp\n", + "Found VRP file: A-n45-k6.vrp\n", + "Found VRP file: A-n80-k10.vrp\n", + "Found VRP file: A-n46-k7.vrp\n", + "Found VRP file: A-n33-k6.vrp\n", + "Found VRP file: A-n37-k5.vrp\n", + "Found VRP file: A-n60-k9.vrp\n", + "Found VRP file: A-n39-k6.vrp\n", + "Found VRP file: A-n32-k5.vrp\n", + "Found VRP file: A-n45-k7.vrp\n", + "Found VRP file: A-n65-k9.vrp\n", + "Found VRP file: A-n64-k9.vrp\n", + "Found VRP file: A-n62-k8.vrp\n", + "Found VRP file: A-n63-k9.vrp\n", + "Found VRP file: A-n54-k7.vrp\n", + "Found VRP file: B-n41-k6.vrp\n", + "Found VRP file: B-n35-k5.vrp\n", + "Found VRP file: B-n38-k6.vrp\n", + "Found VRP file: B-n64-k9.vrp\n", + "Found VRP file: B-n39-k5.vrp\n", + "Found VRP file: B-n50-k8.vrp\n", + "Found VRP file: B-n45-k6.vrp\n", + "Found VRP file: B-n43-k6.vrp\n", + "Found VRP file: B-n45-k5.vrp\n", + "Found VRP file: B-n34-k5.vrp\n", + "Found VRP file: B-n68-k9.vrp\n", + "Found VRP file: B-n56-k7.vrp\n", + "Found VRP file: B-n31-k5.vrp\n", + "Found VRP file: B-n44-k7.vrp\n", + "Found VRP file: B-n66-k9.vrp\n", + "Found VRP file: B-n57-k7.vrp\n", + "Found VRP file: B-n51-k7.vrp\n", + "Found VRP file: B-n50-k7.vrp\n", + "Found VRP file: B-n78-k10.vrp\n", + "Found VRP file: B-n63-k10.vrp\n", + "Found VRP file: B-n67-k10.vrp\n", + "Found VRP file: B-n52-k7.vrp\n", + "Found VRP file: B-n57-k9.vrp\n", + "Found VRP file: E-n101-k8.vrp\n", + "Found VRP file: E-n101-k14.vrp\n", + "Found VRP file: E-n23-k3.vrp\n", + "Found VRP file: E-n22-k4.vrp\n", + "Found VRP file: E-n76-k7.vrp\n", + "Found VRP file: E-n33-k4.vrp\n", + "Found VRP file: E-n13-k4.vrp\n", + "Found VRP file: E-n76-k14.vrp\n", + "Found VRP file: E-n76-k8.vrp\n", + "Found VRP file: E-n76-k10.vrp\n", + "Found VRP file: E-n31-k7.vrp\n", + "Found VRP file: E-n51-k5.vrp\n", + "Found VRP file: E-n30-k3.vrp\n", + "Found VRP file: F-n72-k4.vrp\n", + "Found VRP file: F-n135-k7.vrp\n", + "Found VRP file: F-n45-k4.vrp\n", + "Found VRP file: M-n200-k17.vrp\n", + "Found VRP file: M-n121-k7.vrp\n", + "Found VRP file: M-n200-k16.vrp\n", + "Found VRP file: M-n101-k10.vrp\n", + "Found VRP file: M-n151-k12.vrp\n", + "Found VRP file: P-n22-k2.vrp\n", + "Found VRP file: P-n76-k4.vrp\n", + "Found VRP file: P-n51-k10.vrp\n", + "Found VRP file: P-n55-k7.vrp\n", + "Found VRP file: P-n16-k8.vrp\n", + "Found VRP file: P-n19-k2.vrp\n", + "Found VRP file: P-n60-k15.vrp\n", + "Found VRP file: P-n55-k8.vrp\n", + "Found VRP file: P-n22-k8.vrp\n", + "Found VRP file: P-n40-k5.vrp\n", + "Found VRP file: P-n20-k2.vrp\n", + "Found VRP file: P-n55-k15.vrp\n", + "Found VRP file: P-n65-k10.vrp\n", + "Found VRP file: P-n101-k4.vrp\n", + "Found VRP file: P-n21-k2.vrp\n", + "Found VRP file: P-n50-k8.vrp\n", + "Found VRP file: P-n23-k8.vrp\n", + "Found VRP file: P-n76-k5.vrp\n", + "Found VRP file: P-n70-k10.vrp\n", + "Found VRP file: P-n50-k7.vrp\n", + "Found VRP file: P-n50-k10.vrp\n", + "Found VRP file: P-n45-k5.vrp\n", + "Found VRP file: P-n55-k10.vrp\n", + "Found VRP file: P-n60-k10.vrp\n", + "Found VRP file: X-n120-k6.vrp\n", + "Found VRP file: X-n856-k95.vrp\n", + "Found VRP file: X-n766-k71.vrp\n", + "Found VRP file: X-n524-k153.vrp\n", + "Found VRP file: X-n420-k130.vrp\n", + "Found VRP file: X-n561-k42.vrp\n", + "Found VRP file: X-n214-k11.vrp\n", + "Found VRP file: X-n701-k44.vrp\n", + "Found VRP file: X-n256-k16.vrp\n", + "Found VRP file: X-n125-k30.vrp\n", + "Found VRP file: X-n261-k13.vrp\n", + "Found VRP file: X-n895-k37.vrp\n", + "Found VRP file: X-n819-k171.vrp\n", + "Found VRP file: X-n513-k21.vrp\n", + "Found VRP file: X-n536-k96.vrp\n", + "Found VRP file: X-n344-k43.vrp\n", + "Found VRP file: X-n1001-k43.vrp\n", + "Found VRP file: X-n439-k37.vrp\n", + "Found VRP file: X-n586-k159.vrp\n", + "Found VRP file: X-n401-k29.vrp\n", + "Found VRP file: X-n641-k35.vrp\n", + "Found VRP file: X-n783-k48.vrp\n", + "Found VRP file: X-n480-k70.vrp\n", + "Found VRP file: X-n251-k28.vrp\n", + "Found VRP file: X-n237-k14.vrp\n", + "Found VRP file: X-n613-k62.vrp\n", + "Found VRP file: X-n228-k23.vrp\n", + "Found VRP file: X-n101-k25.vrp\n", + "Found VRP file: X-n336-k84.vrp\n", + "Found VRP file: X-n233-k16.vrp\n", + "Found VRP file: X-n266-k58.vrp\n", + "Found VRP file: X-n294-k50.vrp\n", + "Found VRP file: X-n209-k16.vrp\n", + "Found VRP file: X-n393-k38.vrp\n", + "Found VRP file: X-n469-k138.vrp\n", + "Found VRP file: X-n351-k40.vrp\n", + "Found VRP file: X-n139-k10.vrp\n", + "Found VRP file: X-n289-k60.vrp\n", + "Found VRP file: X-n270-k35.vrp\n", + "Found VRP file: X-n685-k75.vrp\n", + "Found VRP file: X-n599-k92.vrp\n", + "Found VRP file: X-n749-k98.vrp\n", + "Found VRP file: X-n134-k13.vrp\n", + "Found VRP file: X-n548-k50.vrp\n", + "Found VRP file: X-n223-k34.vrp\n", + "Found VRP file: X-n502-k39.vrp\n", + "Found VRP file: X-n106-k14.vrp\n", + "Found VRP file: X-n359-k29.vrp\n", + "Found VRP file: X-n162-k11.vrp\n", + "Found VRP file: X-n115-k10.vrp\n", + "Found VRP file: X-n655-k131.vrp\n", + "Found VRP file: X-n936-k151.vrp\n", + "Found VRP file: X-n573-k30.vrp\n", + "Found VRP file: X-n176-k26.vrp\n", + "Found VRP file: X-n186-k15.vrp\n", + "Found VRP file: X-n376-k94.vrp\n", + "Found VRP file: X-n153-k22.vrp\n", + "Found VRP file: X-n303-k21.vrp\n", + "Found VRP file: X-n167-k10.vrp\n", + "Found VRP file: X-n322-k28.vrp\n", + "Found VRP file: X-n110-k13.vrp\n", + "Found VRP file: X-n449-k29.vrp\n", + "Found VRP file: X-n317-k53.vrp\n", + "Found VRP file: X-n200-k36.vrp\n", + "Found VRP file: X-n143-k7.vrp\n", + "Found VRP file: X-n733-k159.vrp\n", + "Found VRP file: X-n429-k61.vrp\n", + "Found VRP file: X-n157-k13.vrp\n", + "Found VRP file: X-n181-k23.vrp\n", + "Found VRP file: X-n491-k59.vrp\n", + "Found VRP file: X-n204-k19.vrp\n", + "Found VRP file: X-n331-k15.vrp\n", + "Found VRP file: X-n801-k40.vrp\n", + "Found VRP file: X-n172-k51.vrp\n", + "Found VRP file: X-n837-k142.vrp\n", + "Found VRP file: X-n195-k51.vrp\n", + "Found VRP file: X-n670-k130.vrp\n", + "Found VRP file: X-n327-k20.vrp\n", + "Found VRP file: X-n627-k43.vrp\n", + "Found VRP file: X-n298-k31.vrp\n", + "Found VRP file: X-n284-k15.vrp\n", + "Found VRP file: X-n190-k8.vrp\n", + "Found VRP file: X-n148-k46.vrp\n", + "Found VRP file: X-n876-k59.vrp\n", + "Found VRP file: X-n313-k71.vrp\n", + "Found VRP file: X-n280-k17.vrp\n", + "Found VRP file: X-n275-k28.vrp\n", + "Found VRP file: X-n247-k50.vrp\n", + "Found VRP file: X-n219-k73.vrp\n", + "Found VRP file: X-n129-k18.vrp\n", + "Found VRP file: X-n242-k48.vrp\n", + "Found VRP file: X-n411-k19.vrp\n", + "Found VRP file: X-n367-k17.vrp\n", + "Found VRP file: X-n459-k26.vrp\n", + "Found VRP file: X-n979-k58.vrp\n", + "Found VRP file: X-n716-k35.vrp\n", + "Found VRP file: X-n957-k87.vrp\n", + "Found VRP file: X-n384-k52.vrp\n", + "Found VRP file: X-n916-k207.vrp\n", + "Found VRP file: X-n308-k13.vrp\n" + ] + } + ], + "source": [ + "import os\n", + "\n", + "# Ensure you have downloaded vrplib under vrplib/\n", + "# Initialize the instances dictionary\n", + "instances = {}\n", + "\n", + "# Walk through the vrplib directory recursively\n", + "for root, dirs, files in sorted(os.walk('./vrplib')):\n", + " for file in files:\n", + " if file.endswith('.vrp'):\n", + " # Initialize the dictionary for this instance\n", + " instance_name = file[:-4] # Remove the '.vrp' extension\n", + " instances[instance_name] = {\"solution\": None} # Create entry for instance\n", + " # Print the file for verification\n", + " instances[instance_name][\"data\"] = os.path.join(root, file) # Save the VRP file path\n", + " instances[instance_name][\"solution\"] = os.path.join(root, file[:-4] + '.sol') # Save the solution file path\n", + " # ensure the solution file exists\n", + " assert os.path.exists(instances[instance_name][\"solution\"]), f\"Solution file not found for {instance_name}\"\n", + " print(f\"Found VRP file: {file}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "from rl4co.utils.ops import unbatchify, gather_by_index\n", + "\n", + "# Utils function\n", + "def normalize_coord(coord:torch.Tensor) -> torch.Tensor: # if we scale x and y separately, aren't we losing the relative position of the points? i.e. we mess with the distances.\n", + " x, y = coord[:, 0], coord[:, 1]\n", + " x_min, x_max = x.min(), x.max()\n", + " y_min, y_max = y.min(), y.max()\n", + " \n", + " x_scaled = (x - x_min) / (x_max - x_min) \n", + " y_scaled = (y - y_min) / (y_max - y_min)\n", + " coord_scaled = torch.stack([x_scaled, y_scaled], dim=1)\n", + " return coord_scaled \n", + "\n", + "\n", + "def evaluate(model, td, env,\n", + " num_augment=8,\n", + " num_starts=None,\n", + " ):\n", + " \n", + " with torch.inference_mode():\n", + " with torch.amp.autocast(\"cuda\"):\n", + " n_start = env.get_num_starts(td) if num_starts is None else num_starts\n", + "\n", + " if num_augment > 1:\n", + " td = model.augment(td)\n", + "\n", + " # Evaluate policy\n", + " out = model.policy(\n", + " td, env, phase=\"test\", num_starts=n_start, return_actions=True\n", + " )\n", + "\n", + " # Unbatchify reward to [batch_size, num_augment, num_starts].\n", + " reward = unbatchify(out[\"reward\"], (num_augment, n_start))\n", + "\n", + " if n_start > 1:\n", + " # max multi-start reward\n", + " max_reward, max_idxs = reward.max(dim=-1)\n", + " out.update({\"max_reward\": max_reward})\n", + "\n", + " if out.get(\"actions\", None) is not None:\n", + " # Reshape batch to [batch_size, num_augment, num_starts, ...]\n", + " actions = unbatchify(out[\"actions\"], (num_augment, n_start))\n", + " out.update(\n", + " {\"best_multistart_actions\": gather_by_index(actions, max_idxs, dim=max_idxs.dim())}\n", + " )\n", + " out[\"actions\"] = actions\n", + "\n", + " # Get augmentation score only during inference\n", + " if num_augment > 1:\n", + " # If multistart is enabled, we use the best multistart rewards\n", + " reward_ = max_reward if n_start > 1 else reward\n", + " max_aug_reward, max_idxs = reward_.max(dim=1)\n", + " out.update({\"max_aug_reward\": max_aug_reward})\n", + "\n", + " if out.get(\"actions\", None) is not None:\n", + " actions_ = (\n", + " out[\"best_multistart_actions\"] if n_start > 1 else out[\"actions\"]\n", + " )\n", + " out.update({\"best_aug_actions\": gather_by_index(actions_, max_idxs)})\n", + " \n", + " return out\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Problem: A-n37-k6 Cost: 970 Optimal Cost: 949 \t Gap: 2.213%\n", + "Problem: A-n33-k5 Cost: 694 Optimal Cost: 661 \t Gap: 4.992%\n", + "Problem: A-n44-k6 Cost: 988 Optimal Cost: 937 \t Gap: 5.443%\n", + "Problem: A-n48-k7 Cost: 1118 Optimal Cost: 1073 \t Gap: 4.194%\n", + "Problem: A-n39-k5 Cost: 846 Optimal Cost: 822 \t Gap: 2.920%\n", + "Problem: A-n63-k10 Cost: 1331 Optimal Cost: 1314 \t Gap: 1.294%\n", + "Problem: A-n36-k5 Cost: 817 Optimal Cost: 799 \t Gap: 2.253%\n", + "Problem: A-n34-k5 Cost: 787 Optimal Cost: 778 \t Gap: 1.157%\n", + "Problem: A-n55-k9 Cost: 1120 Optimal Cost: 1073 \t Gap: 4.380%\n", + "Problem: A-n69-k9 Cost: 1187 Optimal Cost: 1159 \t Gap: 2.416%\n", + "Problem: A-n61-k9 Cost: 1071 Optimal Cost: 1034 \t Gap: 3.578%\n", + "Problem: A-n38-k5 Cost: 745 Optimal Cost: 730 \t Gap: 2.055%\n", + "Problem: A-n53-k7 Cost: 1052 Optimal Cost: 1010 \t Gap: 4.158%\n", + "Problem: A-n45-k6 Cost: 986 Optimal Cost: 944 \t Gap: 4.449%\n", + "Problem: A-n80-k10 Cost: 1802 Optimal Cost: 1763 \t Gap: 2.212%\n", + "Problem: A-n46-k7 Cost: 924 Optimal Cost: 914 \t Gap: 1.094%\n", + "Problem: A-n33-k6 Cost: 751 Optimal Cost: 742 \t Gap: 1.213%\n", + "Problem: A-n37-k5 Cost: 711 Optimal Cost: 669 \t Gap: 6.278%\n", + "Problem: A-n60-k9 Cost: 1376 Optimal Cost: 1354 \t Gap: 1.625%\n", + "Problem: A-n39-k6 Cost: 871 Optimal Cost: 831 \t Gap: 4.813%\n", + "Problem: A-n32-k5 Cost: 804 Optimal Cost: 784 \t Gap: 2.551%\n", + "Problem: A-n45-k7 Cost: 1156 Optimal Cost: 1146 \t Gap: 0.873%\n", + "Problem: A-n65-k9 Cost: 1200 Optimal Cost: 1174 \t Gap: 2.215%\n", + "Problem: A-n64-k9 Cost: 1434 Optimal Cost: 1401 \t Gap: 2.355%\n", + "Problem: A-n62-k8 Cost: 1323 Optimal Cost: 1288 \t Gap: 2.717%\n", + "Problem: A-n63-k9 Cost: 1649 Optimal Cost: 1616 \t Gap: 2.042%\n", + "Problem: A-n54-k7 Cost: 1176 Optimal Cost: 1167 \t Gap: 0.771%\n", + "Problem: B-n41-k6 Cost: 843 Optimal Cost: 829 \t Gap: 1.689%\n", + "Problem: B-n35-k5 Cost: 975 Optimal Cost: 955 \t Gap: 2.094%\n", + "Problem: B-n38-k6 Cost: 821 Optimal Cost: 805 \t Gap: 1.988%\n", + "Problem: B-n64-k9 Cost: 928 Optimal Cost: 861 \t Gap: 7.782%\n", + "Problem: B-n39-k5 Cost: 555 Optimal Cost: 549 \t Gap: 1.093%\n", + "Problem: B-n50-k8 Cost: 1333 Optimal Cost: 1312 \t Gap: 1.601%\n", + "Problem: B-n45-k6 Cost: 741 Optimal Cost: 678 \t Gap: 9.292%\n", + "Problem: B-n43-k6 Cost: 752 Optimal Cost: 742 \t Gap: 1.348%\n", + "Problem: B-n45-k5 Cost: 771 Optimal Cost: 751 \t Gap: 2.663%\n", + "Problem: B-n34-k5 Cost: 797 Optimal Cost: 788 \t Gap: 1.142%\n", + "Problem: B-n68-k9 Cost: 1296 Optimal Cost: 1272 \t Gap: 1.887%\n", + "Problem: B-n56-k7 Cost: 726 Optimal Cost: 707 \t Gap: 2.687%\n", + "Problem: B-n31-k5 Cost: 689 Optimal Cost: 672 \t Gap: 2.530%\n", + "Problem: B-n44-k7 Cost: 940 Optimal Cost: 909 \t Gap: 3.410%\n", + "Problem: B-n66-k9 Cost: 1342 Optimal Cost: 1316 \t Gap: 1.976%\n", + "Problem: B-n57-k7 Cost: 1156 Optimal Cost: 1153 \t Gap: 0.260%\n", + "Problem: B-n51-k7 Cost: 1025 Optimal Cost: 1032 \t Gap: -0.678%\n", + "Problem: B-n50-k7 Cost: 762 Optimal Cost: 741 \t Gap: 2.834%\n", + "Problem: B-n78-k10 Cost: 1275 Optimal Cost: 1221 \t Gap: 4.423%\n", + "Problem: B-n63-k10 Cost: 1553 Optimal Cost: 1496 \t Gap: 3.810%\n", + "Problem: B-n67-k10 Cost: 1059 Optimal Cost: 1032 \t Gap: 2.616%\n", + "Problem: B-n52-k7 Cost: 765 Optimal Cost: 747 \t Gap: 2.410%\n", + "Problem: B-n57-k9 Cost: 1607 Optimal Cost: 1598 \t Gap: 0.563%\n", + "Problem: E-n101-k8 Cost: 856 Optimal Cost: 815 \t Gap: 5.031%\n", + "Problem: E-n101-k14 Cost: 1126 Optimal Cost: 1067 \t Gap: 5.530%\n", + "Problem: E-n23-k3 Cost: 574 Optimal Cost: 569 \t Gap: 0.879%\n", + "Problem: E-n22-k4 Cost: 376 Optimal Cost: 375 \t Gap: 0.267%\n", + "Problem: E-n76-k7 Cost: 708 Optimal Cost: 682 \t Gap: 3.812%\n", + "Problem: E-n33-k4 Cost: 870 Optimal Cost: 835 \t Gap: 4.192%\n", + "Skipping E-n13-k4 as it does not have node_coord\n", + "Problem: E-n76-k14 Cost: 1052 Optimal Cost: 1021 \t Gap: 3.036%\n", + "Problem: E-n76-k8 Cost: 761 Optimal Cost: 735 \t Gap: 3.537%\n", + "Problem: E-n76-k10 Cost: 858 Optimal Cost: 830 \t Gap: 3.373%\n", + "Skipping E-n31-k7 as it does not have node_coord\n", + "Problem: E-n51-k5 Cost: 549 Optimal Cost: 521 \t Gap: 5.374%\n", + "Problem: E-n30-k3 Cost: 519 Optimal Cost: 534 \t Gap: -2.809%\n", + "Problem: F-n72-k4 Cost: 281 Optimal Cost: 237 \t Gap: 18.565%\n", + "Problem: F-n135-k7 Cost: 1348 Optimal Cost: 1162 \t Gap: 16.007%\n", + "Problem: F-n45-k4 Cost: 755 Optimal Cost: 724 \t Gap: 4.282%\n", + "Problem: M-n200-k17 Cost: 1369 Optimal Cost: 1275 \t Gap: 7.373%\n", + "Problem: M-n121-k7 Cost: 1068 Optimal Cost: 1034 \t Gap: 3.288%\n", + "Problem: M-n200-k16 Cost: 1369 Optimal Cost: 1274 \t Gap: 7.457%\n", + "Problem: M-n101-k10 Cost: 836 Optimal Cost: 820 \t Gap: 1.951%\n", + "Problem: M-n151-k12 Cost: 1069 Optimal Cost: 1015 \t Gap: 5.320%\n", + "Problem: P-n22-k2 Cost: 255 Optimal Cost: 216 \t Gap: 18.056%\n", + "Problem: P-n76-k4 Cost: 635 Optimal Cost: 593 \t Gap: 7.083%\n", + "Problem: P-n51-k10 Cost: 760 Optimal Cost: 741 \t Gap: 2.564%\n", + "Problem: P-n55-k7 Cost: 596 Optimal Cost: 568 \t Gap: 4.930%\n", + "Problem: P-n16-k8 Cost: 452 Optimal Cost: 450 \t Gap: 0.444%\n", + "Problem: P-n19-k2 Cost: 233 Optimal Cost: 212 \t Gap: 9.906%\n", + "Problem: P-n60-k15 Cost: 1004 Optimal Cost: 968 \t Gap: 3.719%\n", + "Problem: P-n55-k8 Cost: 594 Optimal Cost: 588 \t Gap: 1.020%\n", + "Problem: P-n22-k8 Cost: 596 Optimal Cost: 603 \t Gap: -1.161%\n", + "Problem: P-n40-k5 Cost: 488 Optimal Cost: 458 \t Gap: 6.550%\n", + "Problem: P-n20-k2 Cost: 233 Optimal Cost: 216 \t Gap: 7.870%\n", + "Problem: P-n55-k15 Cost: 984 Optimal Cost: 989 \t Gap: -0.506%\n", + "Problem: P-n65-k10 Cost: 808 Optimal Cost: 792 \t Gap: 2.020%\n", + "Problem: P-n101-k4 Cost: 724 Optimal Cost: 681 \t Gap: 6.314%\n", + "Problem: P-n21-k2 Cost: 232 Optimal Cost: 211 \t Gap: 9.953%\n", + "Problem: P-n50-k8 Cost: 658 Optimal Cost: 631 \t Gap: 4.279%\n", + "Problem: P-n23-k8 Cost: 544 Optimal Cost: 529 \t Gap: 2.836%\n", + "Problem: P-n76-k5 Cost: 659 Optimal Cost: 627 \t Gap: 5.104%\n", + "Problem: P-n70-k10 Cost: 850 Optimal Cost: 827 \t Gap: 2.781%\n", + "Problem: P-n50-k7 Cost: 569 Optimal Cost: 554 \t Gap: 2.708%\n", + "Problem: P-n50-k10 Cost: 727 Optimal Cost: 696 \t Gap: 4.454%\n", + "Problem: P-n45-k5 Cost: 526 Optimal Cost: 510 \t Gap: 3.137%\n", + "Problem: P-n55-k10 Cost: 712 Optimal Cost: 694 \t Gap: 2.594%\n", + "Problem: P-n60-k10 Cost: 767 Optimal Cost: 744 \t Gap: 3.091%\n", + "Problem: X-n120-k6 Cost: 13765 Optimal Cost: 13332 \t Gap: 3.248%\n", + "Problem: X-n856-k95 Cost: 98393 Optimal Cost: 88965 \t Gap: 10.597%\n", + "Problem: X-n766-k71 Cost: 130052 Optimal Cost: 114417 \t Gap: 13.665%\n", + "Problem: X-n524-k153 Cost: 174075 Optimal Cost: 154593 \t Gap: 12.602%\n", + "Problem: X-n420-k130 Cost: 116763 Optimal Cost: 107798 \t Gap: 8.316%\n", + "Problem: X-n561-k42 Cost: 49455 Optimal Cost: 42717 \t Gap: 15.774%\n", + "Problem: X-n214-k11 Cost: 11670 Optimal Cost: 10856 \t Gap: 7.498%\n", + "Problem: X-n701-k44 Cost: 90970 Optimal Cost: 81923 \t Gap: 11.043%\n", + "Problem: X-n256-k16 Cost: 19998 Optimal Cost: 18839 \t Gap: 6.152%\n", + "Problem: X-n125-k30 Cost: 58522 Optimal Cost: 55539 \t Gap: 5.371%\n", + "Problem: X-n261-k13 Cost: 28510 Optimal Cost: 26558 \t Gap: 7.350%\n", + "Problem: X-n895-k37 Cost: 64525 Optimal Cost: 53860 \t Gap: 19.801%\n", + "Problem: X-n819-k171 Cost: 174609 Optimal Cost: 158121 \t Gap: 10.427%\n", + "Problem: X-n513-k21 Cost: 28566 Optimal Cost: 24201 \t Gap: 18.036%\n", + "Problem: X-n536-k96 Cost: 103337 Optimal Cost: 94846 \t Gap: 8.952%\n", + "Problem: X-n344-k43 Cost: 44857 Optimal Cost: 42050 \t Gap: 6.675%\n", + "Problem: X-n1001-k43 Cost: 85998 Optimal Cost: 72355 \t Gap: 18.856%\n", + "Problem: X-n439-k37 Cost: 40029 Optimal Cost: 36391 \t Gap: 9.997%\n", + "Problem: X-n586-k159 Cost: 205770 Optimal Cost: 190316 \t Gap: 8.120%\n", + "Problem: X-n401-k29 Cost: 69381 Optimal Cost: 66154 \t Gap: 4.878%\n", + "Problem: X-n641-k35 Cost: 70676 Optimal Cost: 63684 \t Gap: 10.979%\n", + "Problem: X-n783-k48 Cost: 83165 Optimal Cost: 72386 \t Gap: 14.891%\n", + "Problem: X-n480-k70 Cost: 95028 Optimal Cost: 89449 \t Gap: 6.237%\n", + "Problem: X-n251-k28 Cost: 40399 Optimal Cost: 38684 \t Gap: 4.433%\n", + "Problem: X-n237-k14 Cost: 29595 Optimal Cost: 27042 \t Gap: 9.441%\n", + "Problem: X-n613-k62 Cost: 66803 Optimal Cost: 59535 \t Gap: 12.208%\n", + "Problem: X-n228-k23 Cost: 28798 Optimal Cost: 25742 \t Gap: 11.872%\n", + "Problem: X-n101-k25 Cost: 29035 Optimal Cost: 27591 \t Gap: 5.234%\n", + "Problem: X-n336-k84 Cost: 146486 Optimal Cost: 139111 \t Gap: 5.302%\n", + "Problem: X-n233-k16 Cost: 20758 Optimal Cost: 19230 \t Gap: 7.946%\n", + "Problem: X-n266-k58 Cost: 79816 Optimal Cost: 75478 \t Gap: 5.747%\n", + "Problem: X-n294-k50 Cost: 50559 Optimal Cost: 47161 \t Gap: 7.205%\n", + "Problem: X-n209-k16 Cost: 31876 Optimal Cost: 30656 \t Gap: 3.980%\n", + "Problem: X-n393-k38 Cost: 41669 Optimal Cost: 38260 \t Gap: 8.910%\n", + "Problem: X-n469-k138 Cost: 238481 Optimal Cost: 221824 \t Gap: 7.509%\n", + "Problem: X-n351-k40 Cost: 28341 Optimal Cost: 25896 \t Gap: 9.442%\n", + "Problem: X-n139-k10 Cost: 13812 Optimal Cost: 13590 \t Gap: 1.634%\n", + "Problem: X-n289-k60 Cost: 100443 Optimal Cost: 95151 \t Gap: 5.562%\n", + "Problem: X-n270-k35 Cost: 37384 Optimal Cost: 35291 \t Gap: 5.931%\n", + "Problem: X-n685-k75 Cost: 77687 Optimal Cost: 68205 \t Gap: 13.902%\n", + "Problem: X-n599-k92 Cost: 116603 Optimal Cost: 108451 \t Gap: 7.517%\n", + "Problem: X-n749-k98 Cost: 85048 Optimal Cost: 77269 \t Gap: 10.067%\n", + "Problem: X-n134-k13 Cost: 11585 Optimal Cost: 10916 \t Gap: 6.129%\n", + "Problem: X-n548-k50 Cost: 100914 Optimal Cost: 86700 \t Gap: 16.394%\n", + "Problem: X-n223-k34 Cost: 42251 Optimal Cost: 40437 \t Gap: 4.486%\n", + "Problem: X-n502-k39 Cost: 71836 Optimal Cost: 69226 \t Gap: 3.770%\n", + "Problem: X-n106-k14 Cost: 27150 Optimal Cost: 26362 \t Gap: 2.989%\n", + "Problem: X-n359-k29 Cost: 55243 Optimal Cost: 51505 \t Gap: 7.258%\n", + "Problem: X-n162-k11 Cost: 14664 Optimal Cost: 14138 \t Gap: 3.720%\n", + "Problem: X-n115-k10 Cost: 13338 Optimal Cost: 12747 \t Gap: 4.636%\n", + "Problem: X-n655-k131 Cost: 112067 Optimal Cost: 106780 \t Gap: 4.951%\n", + "Problem: X-n936-k151 Cost: 163073 Optimal Cost: 132715 \t Gap: 22.875%\n", + "Problem: X-n573-k30 Cost: 55797 Optimal Cost: 50673 \t Gap: 10.112%\n", + "Problem: X-n176-k26 Cost: 51400 Optimal Cost: 47812 \t Gap: 7.504%\n", + "Problem: X-n186-k15 Cost: 25140 Optimal Cost: 24145 \t Gap: 4.121%\n", + "Problem: X-n376-k94 Cost: 151981 Optimal Cost: 147713 \t Gap: 2.889%\n", + "Problem: X-n153-k22 Cost: 23478 Optimal Cost: 21220 \t Gap: 10.641%\n", + "Problem: X-n303-k21 Cost: 23483 Optimal Cost: 21736 \t Gap: 8.037%\n", + "Problem: X-n167-k10 Cost: 21412 Optimal Cost: 20557 \t Gap: 4.159%\n", + "Problem: X-n322-k28 Cost: 32603 Optimal Cost: 29834 \t Gap: 9.281%\n", + "Problem: X-n110-k13 Cost: 15314 Optimal Cost: 14971 \t Gap: 2.291%\n", + "Problem: X-n449-k29 Cost: 60634 Optimal Cost: 55233 \t Gap: 9.779%\n", + "Problem: X-n317-k53 Cost: 80714 Optimal Cost: 78355 \t Gap: 3.011%\n", + "Problem: X-n200-k36 Cost: 61199 Optimal Cost: 58578 \t Gap: 4.474%\n", + "Problem: X-n143-k7 Cost: 16257 Optimal Cost: 15700 \t Gap: 3.548%\n", + "Problem: X-n733-k159 Cost: 148786 Optimal Cost: 136187 \t Gap: 9.251%\n", + "Problem: X-n429-k61 Cost: 70426 Optimal Cost: 65449 \t Gap: 7.604%\n", + "Problem: X-n157-k13 Cost: 17339 Optimal Cost: 16876 \t Gap: 2.744%\n", + "Problem: X-n181-k23 Cost: 26097 Optimal Cost: 25569 \t Gap: 2.065%\n", + "Problem: X-n491-k59 Cost: 72518 Optimal Cost: 66483 \t Gap: 9.078%\n", + "Problem: X-n204-k19 Cost: 20608 Optimal Cost: 19565 \t Gap: 5.331%\n", + "Problem: X-n331-k15 Cost: 33966 Optimal Cost: 31102 \t Gap: 9.208%\n", + "Problem: X-n801-k40 Cost: 86024 Optimal Cost: 73311 \t Gap: 17.341%\n", + "Problem: X-n172-k51 Cost: 48118 Optimal Cost: 45607 \t Gap: 5.506%\n", + "Problem: X-n837-k142 Cost: 208252 Optimal Cost: 193737 \t Gap: 7.492%\n", + "Problem: X-n195-k51 Cost: 47390 Optimal Cost: 44225 \t Gap: 7.157%\n", + "Problem: X-n670-k130 Cost: 169056 Optimal Cost: 146332 \t Gap: 15.529%\n", + "Problem: X-n327-k20 Cost: 29784 Optimal Cost: 27532 \t Gap: 8.180%\n", + "Problem: X-n627-k43 Cost: 67339 Optimal Cost: 62164 \t Gap: 8.325%\n", + "Problem: X-n298-k31 Cost: 36706 Optimal Cost: 34231 \t Gap: 7.230%\n", + "Problem: X-n284-k15 Cost: 22043 Optimal Cost: 20226 \t Gap: 8.983%\n", + "Problem: X-n190-k8 Cost: 17892 Optimal Cost: 16980 \t Gap: 5.371%\n", + "Problem: X-n148-k46 Cost: 45036 Optimal Cost: 43448 \t Gap: 3.655%\n", + "Problem: X-n876-k59 Cost: 107229 Optimal Cost: 99299 \t Gap: 7.986%\n", + "Problem: X-n313-k71 Cost: 99661 Optimal Cost: 94043 \t Gap: 5.974%\n", + "Problem: X-n280-k17 Cost: 36549 Optimal Cost: 33503 \t Gap: 9.092%\n", + "Problem: X-n275-k28 Cost: 24187 Optimal Cost: 21245 \t Gap: 13.848%\n", + "Problem: X-n247-k50 Cost: 40639 Optimal Cost: 37274 \t Gap: 9.028%\n", + "Problem: X-n219-k73 Cost: 120348 Optimal Cost: 117595 \t Gap: 2.341%\n", + "Problem: X-n129-k18 Cost: 29598 Optimal Cost: 28940 \t Gap: 2.274%\n", + "Problem: X-n242-k48 Cost: 85704 Optimal Cost: 82751 \t Gap: 3.569%\n", + "Problem: X-n411-k19 Cost: 22868 Optimal Cost: 19712 \t Gap: 16.011%\n", + "Problem: X-n367-k17 Cost: 25578 Optimal Cost: 22814 \t Gap: 12.115%\n", + "Problem: X-n459-k26 Cost: 27341 Optimal Cost: 24139 \t Gap: 13.265%\n", + "Problem: X-n979-k58 Cost: 129770 Optimal Cost: 118976 \t Gap: 9.072%\n", + "Problem: X-n716-k35 Cost: 49709 Optimal Cost: 43373 \t Gap: 14.608%\n", + "Problem: X-n957-k87 Cost: 102964 Optimal Cost: 85465 \t Gap: 20.475%\n", + "Problem: X-n384-k52 Cost: 70231 Optimal Cost: 65940 \t Gap: 6.507%\n", + "Problem: X-n916-k207 Cost: 352732 Optimal Cost: 329179 \t Gap: 7.155%\n", + "Problem: X-n308-k13 Cost: 28434 Optimal Cost: 25859 \t Gap: 9.958%\n" + ] + } + ], + "source": [ + "from tensordict import TensorDict\n", + "from math import ceil\n", + "import time\n", + "import vrplib\n", + "\n", + "\n", + "model.eval().to(device)\n", + "\n", + "results = []\n", + "\n", + "for instance in instances:\n", + " problem = vrplib.read_instance(instances[instance][\"data\"])\n", + "\n", + " if problem.get(\"node_coord\", None) is None:\n", + " print(f\"Skipping {instance} as it does not have node_coord\")\n", + " continue\n", + " coords = torch.tensor(problem['node_coord']).float()\n", + " coords_norm = normalize_coord(coords)\n", + "\n", + " original_capacity = problem['capacity']\n", + " demand = torch.tensor(problem['demand'][1:]).float() / original_capacity\n", + " original_capacity = torch.tensor(original_capacity)[None] \n", + "\n", + " # Make instance\n", + " td_instance = TensorDict({\n", + " \"locs\": coords_norm,\n", + " \"demand_linehaul\": demand,\n", + " \"capacity_original\": original_capacity,\n", + " },\n", + " batch_size=[])[None]\n", + "\n", + " td_reset = env.reset(td_instance).to(device)\n", + " \n", + " start = time.time()\n", + " actions = evaluate(model, td_reset.clone(), env)[\"best_aug_actions\"]\n", + " inference_time = time.time() - start \n", + "\n", + " # Obtain reward from the environment with new locs\n", + " td_reset[\"locs\"] = coords[None] # unnormalized\n", + " reward = env.get_reward(td_reset, actions)\n", + " \n", + " # Load the optimal cost\n", + " solution = vrplib.read_solution(instances[instance][\"solution\"])\n", + " optimal_cost = solution['cost'] # note that this cost is somehow slightly lower than the one calculated from the distance matrix\n", + " \n", + " # Calculate the gap and print\n", + " cost = ceil(-reward.item())\n", + " gap = (cost - optimal_cost) / optimal_cost\n", + " print(f'Problem: {instance:<15} Cost: {cost:<10} Optimal Cost: {optimal_cost:<10}\\t Gap: {gap:.3%}')\n", + " \n", + " results.append({\n", + " \"instance\": instance,\n", + " \"cost\": cost,\n", + " \"optimal_cost\": optimal_cost,\n", + " \"gap\": gap,\n", + " \"inference_time\": inference_time,\n", + " })" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# Save the results with pkl\n", + "import pickle\n", + "import os \n", + "\n", + "SAVEDIR = \"results/cvrplib/\"\n", + "os.makedirs(SAVEDIR, exist_ok=True)\n", + "\n", + "# TODO: change the filename\n", + "with open(SAVEDIR+'rf-transformer.pkl', 'wb') as f:\n", + " pickle.dump(results, f)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Load data" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Problem: A-n32-k5 Cost: 804 Optimal Cost: 784 \t Gap: 2.551%\n", + "Problem: A-n33-k5 Cost: 694 Optimal Cost: 661 \t Gap: 4.992%\n", + "Problem: A-n33-k6 Cost: 751 Optimal Cost: 742 \t Gap: 1.213%\n", + "Problem: A-n34-k5 Cost: 787 Optimal Cost: 778 \t Gap: 1.157%\n", + "Problem: A-n36-k5 Cost: 817 Optimal Cost: 799 \t Gap: 2.253%\n", + "Problem: A-n37-k5 Cost: 711 Optimal Cost: 669 \t Gap: 6.278%\n", + "Problem: A-n37-k6 Cost: 970 Optimal Cost: 949 \t Gap: 2.213%\n", + "Problem: A-n38-k5 Cost: 745 Optimal Cost: 730 \t Gap: 2.055%\n", + "Problem: A-n39-k5 Cost: 846 Optimal Cost: 822 \t Gap: 2.920%\n", + "Problem: A-n39-k6 Cost: 871 Optimal Cost: 831 \t Gap: 4.813%\n", + "Problem: A-n44-k6 Cost: 988 Optimal Cost: 937 \t Gap: 5.443%\n", + "Problem: A-n45-k6 Cost: 986 Optimal Cost: 944 \t Gap: 4.449%\n", + "Problem: A-n45-k7 Cost: 1156 Optimal Cost: 1146 \t Gap: 0.873%\n", + "Problem: A-n46-k7 Cost: 924 Optimal Cost: 914 \t Gap: 1.094%\n", + "Problem: A-n48-k7 Cost: 1118 Optimal Cost: 1073 \t Gap: 4.194%\n", + "Problem: A-n53-k7 Cost: 1052 Optimal Cost: 1010 \t Gap: 4.158%\n", + "Problem: A-n54-k7 Cost: 1176 Optimal Cost: 1167 \t Gap: 0.771%\n", + "Problem: A-n55-k9 Cost: 1120 Optimal Cost: 1073 \t Gap: 4.380%\n", + "Problem: A-n60-k9 Cost: 1376 Optimal Cost: 1354 \t Gap: 1.625%\n", + "Problem: A-n61-k9 Cost: 1071 Optimal Cost: 1034 \t Gap: 3.578%\n", + "Problem: A-n62-k8 Cost: 1323 Optimal Cost: 1288 \t Gap: 2.717%\n", + "Problem: A-n63-k10 Cost: 1331 Optimal Cost: 1314 \t Gap: 1.294%\n", + "Problem: A-n63-k9 Cost: 1649 Optimal Cost: 1616 \t Gap: 2.042%\n", + "Problem: A-n64-k9 Cost: 1434 Optimal Cost: 1401 \t Gap: 2.355%\n", + "Problem: A-n65-k9 Cost: 1200 Optimal Cost: 1174 \t Gap: 2.215%\n", + "Problem: A-n69-k9 Cost: 1187 Optimal Cost: 1159 \t Gap: 2.416%\n", + "Problem: A-n80-k10 Cost: 1802 Optimal Cost: 1763 \t Gap: 2.212%\n", + "Problem: B-n31-k5 Cost: 689 Optimal Cost: 672 \t Gap: 2.530%\n", + "Problem: B-n34-k5 Cost: 797 Optimal Cost: 788 \t Gap: 1.142%\n", + "Problem: B-n35-k5 Cost: 975 Optimal Cost: 955 \t Gap: 2.094%\n", + "Problem: B-n38-k6 Cost: 821 Optimal Cost: 805 \t Gap: 1.988%\n", + "Problem: B-n39-k5 Cost: 555 Optimal Cost: 549 \t Gap: 1.093%\n", + "Problem: B-n41-k6 Cost: 843 Optimal Cost: 829 \t Gap: 1.689%\n", + "Problem: B-n43-k6 Cost: 752 Optimal Cost: 742 \t Gap: 1.348%\n", + "Problem: B-n44-k7 Cost: 940 Optimal Cost: 909 \t Gap: 3.410%\n", + "Problem: B-n45-k5 Cost: 771 Optimal Cost: 751 \t Gap: 2.663%\n", + "Problem: B-n45-k6 Cost: 741 Optimal Cost: 678 \t Gap: 9.292%\n", + "Problem: B-n50-k7 Cost: 762 Optimal Cost: 741 \t Gap: 2.834%\n", + "Problem: B-n50-k8 Cost: 1333 Optimal Cost: 1312 \t Gap: 1.601%\n", + "Problem: B-n51-k7 Cost: 1025 Optimal Cost: 1032 \t Gap: -0.678%\n", + "Problem: B-n52-k7 Cost: 765 Optimal Cost: 747 \t Gap: 2.410%\n", + "Problem: B-n56-k7 Cost: 726 Optimal Cost: 707 \t Gap: 2.687%\n", + "Problem: B-n57-k7 Cost: 1156 Optimal Cost: 1153 \t Gap: 0.260%\n", + "Problem: B-n57-k9 Cost: 1607 Optimal Cost: 1598 \t Gap: 0.563%\n", + "Problem: B-n63-k10 Cost: 1553 Optimal Cost: 1496 \t Gap: 3.810%\n", + "Problem: B-n64-k9 Cost: 928 Optimal Cost: 861 \t Gap: 7.782%\n", + "Problem: B-n66-k9 Cost: 1342 Optimal Cost: 1316 \t Gap: 1.976%\n", + "Problem: B-n67-k10 Cost: 1059 Optimal Cost: 1032 \t Gap: 2.616%\n", + "Problem: B-n68-k9 Cost: 1296 Optimal Cost: 1272 \t Gap: 1.887%\n", + "Problem: B-n78-k10 Cost: 1275 Optimal Cost: 1221 \t Gap: 4.423%\n", + "Problem: E-n101-k14 Cost: 1126 Optimal Cost: 1067 \t Gap: 5.530%\n", + "Problem: E-n101-k8 Cost: 856 Optimal Cost: 815 \t Gap: 5.031%\n", + "Problem: E-n22-k4 Cost: 376 Optimal Cost: 375 \t Gap: 0.267%\n", + "Problem: E-n23-k3 Cost: 574 Optimal Cost: 569 \t Gap: 0.879%\n", + "Problem: E-n30-k3 Cost: 519 Optimal Cost: 534 \t Gap: -2.809%\n", + "Problem: E-n33-k4 Cost: 870 Optimal Cost: 835 \t Gap: 4.192%\n", + "Problem: E-n51-k5 Cost: 549 Optimal Cost: 521 \t Gap: 5.374%\n", + "Problem: E-n76-k10 Cost: 858 Optimal Cost: 830 \t Gap: 3.373%\n", + "Problem: E-n76-k14 Cost: 1052 Optimal Cost: 1021 \t Gap: 3.036%\n", + "Problem: E-n76-k7 Cost: 708 Optimal Cost: 682 \t Gap: 3.812%\n", + "Problem: E-n76-k8 Cost: 761 Optimal Cost: 735 \t Gap: 3.537%\n", + "Problem: F-n135-k7 Cost: 1348 Optimal Cost: 1162 \t Gap: 16.007%\n", + "Problem: F-n45-k4 Cost: 755 Optimal Cost: 724 \t Gap: 4.282%\n", + "Problem: F-n72-k4 Cost: 281 Optimal Cost: 237 \t Gap: 18.565%\n", + "Problem: M-n101-k10 Cost: 836 Optimal Cost: 820 \t Gap: 1.951%\n", + "Problem: M-n121-k7 Cost: 1068 Optimal Cost: 1034 \t Gap: 3.288%\n", + "Problem: M-n151-k12 Cost: 1069 Optimal Cost: 1015 \t Gap: 5.320%\n", + "Problem: M-n200-k16 Cost: 1369 Optimal Cost: 1274 \t Gap: 7.457%\n", + "Problem: M-n200-k17 Cost: 1369 Optimal Cost: 1275 \t Gap: 7.373%\n", + "Problem: P-n101-k4 Cost: 724 Optimal Cost: 681 \t Gap: 6.314%\n", + "Problem: P-n16-k8 Cost: 452 Optimal Cost: 450 \t Gap: 0.444%\n", + "Problem: P-n19-k2 Cost: 233 Optimal Cost: 212 \t Gap: 9.906%\n", + "Problem: P-n20-k2 Cost: 233 Optimal Cost: 216 \t Gap: 7.870%\n", + "Problem: P-n21-k2 Cost: 232 Optimal Cost: 211 \t Gap: 9.953%\n", + "Problem: P-n22-k2 Cost: 255 Optimal Cost: 216 \t Gap: 18.056%\n", + "Problem: P-n22-k8 Cost: 596 Optimal Cost: 603 \t Gap: -1.161%\n", + "Problem: P-n23-k8 Cost: 544 Optimal Cost: 529 \t Gap: 2.836%\n", + "Problem: P-n40-k5 Cost: 488 Optimal Cost: 458 \t Gap: 6.550%\n", + "Problem: P-n45-k5 Cost: 526 Optimal Cost: 510 \t Gap: 3.137%\n", + "Problem: P-n50-k10 Cost: 727 Optimal Cost: 696 \t Gap: 4.454%\n", + "Problem: P-n50-k7 Cost: 569 Optimal Cost: 554 \t Gap: 2.708%\n", + "Problem: P-n50-k8 Cost: 658 Optimal Cost: 631 \t Gap: 4.279%\n", + "Problem: P-n51-k10 Cost: 760 Optimal Cost: 741 \t Gap: 2.564%\n", + "Problem: P-n55-k10 Cost: 712 Optimal Cost: 694 \t Gap: 2.594%\n", + "Problem: P-n55-k15 Cost: 984 Optimal Cost: 989 \t Gap: -0.506%\n", + "Problem: P-n55-k7 Cost: 596 Optimal Cost: 568 \t Gap: 4.930%\n", + "Problem: P-n55-k8 Cost: 594 Optimal Cost: 588 \t Gap: 1.020%\n", + "Problem: P-n60-k10 Cost: 767 Optimal Cost: 744 \t Gap: 3.091%\n", + "Problem: P-n60-k15 Cost: 1004 Optimal Cost: 968 \t Gap: 3.719%\n", + "Problem: P-n65-k10 Cost: 808 Optimal Cost: 792 \t Gap: 2.020%\n", + "Problem: P-n70-k10 Cost: 850 Optimal Cost: 827 \t Gap: 2.781%\n", + "Problem: P-n76-k4 Cost: 635 Optimal Cost: 593 \t Gap: 7.083%\n", + "Problem: P-n76-k5 Cost: 659 Optimal Cost: 627 \t Gap: 5.104%\n", + "Problem: X-n1001-k43 Cost: 85998 Optimal Cost: 72355 \t Gap: 18.856%\n", + "Problem: X-n101-k25 Cost: 29035 Optimal Cost: 27591 \t Gap: 5.234%\n", + "Problem: X-n106-k14 Cost: 27150 Optimal Cost: 26362 \t Gap: 2.989%\n", + "Problem: X-n110-k13 Cost: 15314 Optimal Cost: 14971 \t Gap: 2.291%\n", + "Problem: X-n115-k10 Cost: 13338 Optimal Cost: 12747 \t Gap: 4.636%\n", + "Problem: X-n120-k6 Cost: 13765 Optimal Cost: 13332 \t Gap: 3.248%\n", + "Problem: X-n125-k30 Cost: 58522 Optimal Cost: 55539 \t Gap: 5.371%\n", + "Problem: X-n129-k18 Cost: 29598 Optimal Cost: 28940 \t Gap: 2.274%\n", + "Problem: X-n134-k13 Cost: 11585 Optimal Cost: 10916 \t Gap: 6.129%\n", + "Problem: X-n139-k10 Cost: 13812 Optimal Cost: 13590 \t Gap: 1.634%\n", + "Problem: X-n143-k7 Cost: 16257 Optimal Cost: 15700 \t Gap: 3.548%\n", + "Problem: X-n148-k46 Cost: 45036 Optimal Cost: 43448 \t Gap: 3.655%\n", + "Problem: X-n153-k22 Cost: 23478 Optimal Cost: 21220 \t Gap: 10.641%\n", + "Problem: X-n157-k13 Cost: 17339 Optimal Cost: 16876 \t Gap: 2.744%\n", + "Problem: X-n162-k11 Cost: 14664 Optimal Cost: 14138 \t Gap: 3.720%\n", + "Problem: X-n167-k10 Cost: 21412 Optimal Cost: 20557 \t Gap: 4.159%\n", + "Problem: X-n172-k51 Cost: 48118 Optimal Cost: 45607 \t Gap: 5.506%\n", + "Problem: X-n176-k26 Cost: 51400 Optimal Cost: 47812 \t Gap: 7.504%\n", + "Problem: X-n181-k23 Cost: 26097 Optimal Cost: 25569 \t Gap: 2.065%\n", + "Problem: X-n186-k15 Cost: 25140 Optimal Cost: 24145 \t Gap: 4.121%\n", + "Problem: X-n190-k8 Cost: 17892 Optimal Cost: 16980 \t Gap: 5.371%\n", + "Problem: X-n195-k51 Cost: 47390 Optimal Cost: 44225 \t Gap: 7.157%\n", + "Problem: X-n200-k36 Cost: 61199 Optimal Cost: 58578 \t Gap: 4.474%\n", + "Problem: X-n204-k19 Cost: 20608 Optimal Cost: 19565 \t Gap: 5.331%\n", + "Problem: X-n209-k16 Cost: 31876 Optimal Cost: 30656 \t Gap: 3.980%\n", + "Problem: X-n214-k11 Cost: 11670 Optimal Cost: 10856 \t Gap: 7.498%\n", + "Problem: X-n219-k73 Cost: 120348 Optimal Cost: 117595 \t Gap: 2.341%\n", + "Problem: X-n223-k34 Cost: 42251 Optimal Cost: 40437 \t Gap: 4.486%\n", + "Problem: X-n228-k23 Cost: 28798 Optimal Cost: 25742 \t Gap: 11.872%\n", + "Problem: X-n233-k16 Cost: 20758 Optimal Cost: 19230 \t Gap: 7.946%\n", + "Problem: X-n237-k14 Cost: 29595 Optimal Cost: 27042 \t Gap: 9.441%\n", + "Problem: X-n242-k48 Cost: 85704 Optimal Cost: 82751 \t Gap: 3.569%\n", + "Problem: X-n247-k50 Cost: 40639 Optimal Cost: 37274 \t Gap: 9.028%\n", + "Problem: X-n251-k28 Cost: 40399 Optimal Cost: 38684 \t Gap: 4.433%\n", + "Problem: X-n256-k16 Cost: 19998 Optimal Cost: 18839 \t Gap: 6.152%\n", + "Problem: X-n261-k13 Cost: 28510 Optimal Cost: 26558 \t Gap: 7.350%\n", + "Problem: X-n266-k58 Cost: 79816 Optimal Cost: 75478 \t Gap: 5.747%\n", + "Problem: X-n270-k35 Cost: 37384 Optimal Cost: 35291 \t Gap: 5.931%\n", + "Problem: X-n275-k28 Cost: 24187 Optimal Cost: 21245 \t Gap: 13.848%\n", + "Problem: X-n280-k17 Cost: 36549 Optimal Cost: 33503 \t Gap: 9.092%\n", + "Problem: X-n284-k15 Cost: 22043 Optimal Cost: 20226 \t Gap: 8.983%\n", + "Problem: X-n289-k60 Cost: 100443 Optimal Cost: 95151 \t Gap: 5.562%\n", + "Problem: X-n294-k50 Cost: 50559 Optimal Cost: 47161 \t Gap: 7.205%\n", + "Problem: X-n298-k31 Cost: 36706 Optimal Cost: 34231 \t Gap: 7.230%\n", + "Problem: X-n303-k21 Cost: 23483 Optimal Cost: 21736 \t Gap: 8.037%\n", + "Problem: X-n308-k13 Cost: 28434 Optimal Cost: 25859 \t Gap: 9.958%\n", + "Problem: X-n313-k71 Cost: 99661 Optimal Cost: 94043 \t Gap: 5.974%\n", + "Problem: X-n317-k53 Cost: 80714 Optimal Cost: 78355 \t Gap: 3.011%\n", + "Problem: X-n322-k28 Cost: 32603 Optimal Cost: 29834 \t Gap: 9.281%\n", + "Problem: X-n327-k20 Cost: 29784 Optimal Cost: 27532 \t Gap: 8.180%\n", + "Problem: X-n331-k15 Cost: 33966 Optimal Cost: 31102 \t Gap: 9.208%\n", + "Problem: X-n336-k84 Cost: 146486 Optimal Cost: 139111 \t Gap: 5.302%\n", + "Problem: X-n344-k43 Cost: 44857 Optimal Cost: 42050 \t Gap: 6.675%\n", + "Problem: X-n351-k40 Cost: 28341 Optimal Cost: 25896 \t Gap: 9.442%\n", + "Problem: X-n359-k29 Cost: 55243 Optimal Cost: 51505 \t Gap: 7.258%\n", + "Problem: X-n367-k17 Cost: 25578 Optimal Cost: 22814 \t Gap: 12.115%\n", + "Problem: X-n376-k94 Cost: 151981 Optimal Cost: 147713 \t Gap: 2.889%\n", + "Problem: X-n384-k52 Cost: 70231 Optimal Cost: 65940 \t Gap: 6.507%\n", + "Problem: X-n393-k38 Cost: 41669 Optimal Cost: 38260 \t Gap: 8.910%\n", + "Problem: X-n401-k29 Cost: 69381 Optimal Cost: 66154 \t Gap: 4.878%\n", + "Problem: X-n411-k19 Cost: 22868 Optimal Cost: 19712 \t Gap: 16.011%\n", + "Problem: X-n420-k130 Cost: 116763 Optimal Cost: 107798 \t Gap: 8.316%\n", + "Problem: X-n429-k61 Cost: 70426 Optimal Cost: 65449 \t Gap: 7.604%\n", + "Problem: X-n439-k37 Cost: 40029 Optimal Cost: 36391 \t Gap: 9.997%\n", + "Problem: X-n449-k29 Cost: 60634 Optimal Cost: 55233 \t Gap: 9.779%\n", + "Problem: X-n459-k26 Cost: 27341 Optimal Cost: 24139 \t Gap: 13.265%\n", + "Problem: X-n469-k138 Cost: 238481 Optimal Cost: 221824 \t Gap: 7.509%\n", + "Problem: X-n480-k70 Cost: 95028 Optimal Cost: 89449 \t Gap: 6.237%\n", + "Problem: X-n491-k59 Cost: 72518 Optimal Cost: 66483 \t Gap: 9.078%\n", + "Problem: X-n502-k39 Cost: 71836 Optimal Cost: 69226 \t Gap: 3.770%\n", + "Problem: X-n513-k21 Cost: 28566 Optimal Cost: 24201 \t Gap: 18.036%\n", + "Problem: X-n524-k153 Cost: 174075 Optimal Cost: 154593 \t Gap: 12.602%\n", + "Problem: X-n536-k96 Cost: 103337 Optimal Cost: 94846 \t Gap: 8.952%\n", + "Problem: X-n548-k50 Cost: 100914 Optimal Cost: 86700 \t Gap: 16.394%\n", + "Problem: X-n561-k42 Cost: 49455 Optimal Cost: 42717 \t Gap: 15.774%\n", + "Problem: X-n573-k30 Cost: 55797 Optimal Cost: 50673 \t Gap: 10.112%\n", + "Problem: X-n586-k159 Cost: 205770 Optimal Cost: 190316 \t Gap: 8.120%\n", + "Problem: X-n599-k92 Cost: 116603 Optimal Cost: 108451 \t Gap: 7.517%\n", + "Problem: X-n613-k62 Cost: 66803 Optimal Cost: 59535 \t Gap: 12.208%\n", + "Problem: X-n627-k43 Cost: 67339 Optimal Cost: 62164 \t Gap: 8.325%\n", + "Problem: X-n641-k35 Cost: 70676 Optimal Cost: 63684 \t Gap: 10.979%\n", + "Problem: X-n655-k131 Cost: 112067 Optimal Cost: 106780 \t Gap: 4.951%\n", + "Problem: X-n670-k130 Cost: 169056 Optimal Cost: 146332 \t Gap: 15.529%\n", + "Problem: X-n685-k75 Cost: 77687 Optimal Cost: 68205 \t Gap: 13.902%\n", + "Problem: X-n701-k44 Cost: 90970 Optimal Cost: 81923 \t Gap: 11.043%\n", + "Problem: X-n716-k35 Cost: 49709 Optimal Cost: 43373 \t Gap: 14.608%\n", + "Problem: X-n733-k159 Cost: 148786 Optimal Cost: 136187 \t Gap: 9.251%\n", + "Problem: X-n749-k98 Cost: 85048 Optimal Cost: 77269 \t Gap: 10.067%\n", + "Problem: X-n766-k71 Cost: 130052 Optimal Cost: 114417 \t Gap: 13.665%\n", + "Problem: X-n783-k48 Cost: 83165 Optimal Cost: 72386 \t Gap: 14.891%\n", + "Problem: X-n801-k40 Cost: 86024 Optimal Cost: 73311 \t Gap: 17.341%\n", + "Problem: X-n819-k171 Cost: 174609 Optimal Cost: 158121 \t Gap: 10.427%\n", + "Problem: X-n837-k142 Cost: 208252 Optimal Cost: 193737 \t Gap: 7.492%\n", + "Problem: X-n856-k95 Cost: 98393 Optimal Cost: 88965 \t Gap: 10.597%\n", + "Problem: X-n876-k59 Cost: 107229 Optimal Cost: 99299 \t Gap: 7.986%\n", + "Problem: X-n895-k37 Cost: 64525 Optimal Cost: 53860 \t Gap: 19.801%\n", + "Problem: X-n916-k207 Cost: 352732 Optimal Cost: 329179 \t Gap: 7.155%\n", + "Problem: X-n936-k151 Cost: 163073 Optimal Cost: 132715 \t Gap: 22.875%\n", + "Problem: X-n957-k87 Cost: 102964 Optimal Cost: 85465 \t Gap: 20.475%\n", + "Problem: X-n979-k58 Cost: 129770 Optimal Cost: 118976 \t Gap: 9.072%\n" + ] + } + ], + "source": [ + "import pickle\n", + "\n", + "# Load\n", + "with open(SAVEDIR+'rf-transformer.pkl', 'rb') as f:\n", + " results = pickle.load(f)\n", + "\n", + "# sort results by name\n", + "results = sorted(results, key=lambda x: x[\"instance\"])\n", + "\n", + "for instance in results:\n", + " print(f'Problem: {instance[\"instance\"]:<15} Cost: {instance[\"cost\"]:<10} Optimal Cost: {instance[\"optimal_cost\"]:<10}\\t Gap: {instance[\"gap\"]:.3%}')" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Group A: 2.825% %\n", + "Group B: 2.583% %\n", + "Group E: 2.929% %\n", + "Group F: 12.951% %\n", + "Group M: 5.078% %\n", + "Group P: 4.573% %\n", + "Group X: 8.437% %\n" + ] + } + ], + "source": [ + "# Take the results. Measure gap depending on first letter of the instance name\n", + "gaps_names = {}\n", + "for instance in results:\n", + " name = instance['instance'][0]\n", + " # if name == 'X':\n", + " # num_locs = instance['instance'].split('-')[1][1:]\n", + " # if int(num_locs) < 252:\n", + " # name = 'X<251'\n", + " # elif int(num_locs) >= 252 and int(num_locs) <= 501:\n", + " # name = 'X251-501'\n", + " # else:\n", + " # name = 'X501-1000'\n", + " if name not in gaps_names:\n", + " # if X, then we divide between < 251 and >= 251\n", + " gaps_names[name] = []\n", + " gaps_names[name].append(instance['gap'])\n", + " \n", + "# Calculate the average gap for each group\n", + "average_gaps = {name: sum(gaps) / len(gaps) for name, gaps in gaps_names.items()}\n", + "for name, gap in average_gaps.items():\n", + " print(f'Group {name}: {(gap):.3%} %')" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Group A: 2.825% %\n", + "Group B: 2.583% %\n", + "Group E: 2.929% %\n", + "Group F: 12.951% %\n", + "Group M: 5.078% %\n", + "Group P: 4.573% %\n", + "Group X501-1000: 12.274% %\n", + "Group X<251: 5.103% %\n", + "Group X251-501: 8.072% %\n" + ] + } + ], + "source": [ + "# Take the results. Measure gap depending on first letter of the instance name\n", + "gaps_names = {}\n", + "for instance in results:\n", + " name = instance['instance'][0]\n", + " if name == 'X':\n", + " num_locs = instance['instance'].split('-')[1][1:]\n", + " if int(num_locs) < 252:\n", + " name = 'X<251'\n", + " elif int(num_locs) >= 252 and int(num_locs) <= 501:\n", + " name = 'X251-501'\n", + " else:\n", + " name = 'X501-1000'\n", + " if name not in gaps_names:\n", + " # if X, then we divide between < 251 and >= 251\n", + " gaps_names[name] = []\n", + " gaps_names[name].append(instance['gap'])\n", + " \n", + "# Calculate the average gap for each group\n", + "average_gaps = {name: sum(gaps) / len(gaps) for name, gaps in gaps_names.items()}\n", + "for name, gap in average_gaps.items():\n", + " print(f'Group {name}: {(gap):.3%} %')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "rl4co", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}