4_model_Combination.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AhmadHossain8/Food-Image-Classification/blob/main/4_model_Combination.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WMqK9WJu0rr3"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from torch.utils.data import Dataset\n",
        "from torch.optim.swa_utils import AveragedModel, SWALR\n",
        "import numpy as np\n",
        "import copy\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "import zipfile\n",
        "\n",
        "DATA_SET_ZIP = '/content/drive/MyDrive/Food/Dataset.zip'\n",
        "with zipfile.ZipFile(DATA_SET_ZIP,\"r\") as zip_ref:\n",
        "  zip_ref.extractall(\"data\")\n",
        "\n",
        "import os\n",
        "import time\n",
        "import pandas as pd\n",
        "from PIL import Image\n",
        "from torchvision import transforms,models\n",
        "import matplotlib.pyplot as plt\n",
        "from skimage import io,transform\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "syRao_530xZb"
      },
      "outputs": [],
      "source": [
        "path_of_training_data = '/content/data/training'\n",
        "path_of_evaluation_data = '/content/data/evaluation'\n",
        "path_of_validation_data = '/content/data/validation'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ujO958En03o6"
      },
      "outputs": [],
      "source": [
        "class Food_11_Dataset(Dataset):\n",
        "  def __init__(self,data_dir, transform=None):\n",
        "    self.date_dir = data_dir\n",
        "    self.image_list = os.listdir(data_dir)\n",
        "    self.transform = transform\n",
        "\n",
        "  def __getitem__(self,idx):\n",
        "    location = os.path.join(self.date_dir,self.image_list[idx])\n",
        "    image = Image.open(location)\n",
        "    if self.transform is not None:\n",
        "            image = self.transform(image)\n",
        "    label = self.image_list[idx]\n",
        "    label = label.split(\"_\")\n",
        "    label = int(label[0])\n",
        "    return image,label\n",
        "\n",
        "  def __len__(self):\n",
        "    return len(self.image_list)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GjD1D5kD06Dm"
      },
      "outputs": [],
      "source": [
        "LEARNING_RATE = 0.001\n",
        "NUM_EPOCHS = 10\n",
        "BATCH_SIZE = 10\n",
        "NUM_CLASSES = 11\n",
        "\n",
        "transform1 = transforms.Compose([\n",
        "    transforms.Resize((227,227)),\n",
        "    transforms.RandomHorizontalFlip(),\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize(mean=[0.50,0.50 ,0.50], std=[0.50, 0.50, 0.50]),\n",
        "])\n",
        "\n",
        "transform2 = transforms.Compose([\n",
        "    transforms.Resize((299,299)),\n",
        "    transforms.RandomHorizontalFlip(),\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize(mean=[0.50,0.50 ,0.50], std=[0.50, 0.50, 0.50]),\n",
        "])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rZxV--3o0_NC"
      },
      "outputs": [],
      "source": [
        "trainset1 = Food_11_Dataset(path_of_training_data,transform=transform1)\n",
        "trainset_loader1 = torch.utils.data.DataLoader(trainset1, batch_size=BATCH_SIZE, shuffle=False)\n",
        "\n",
        "evaluationset1 = Food_11_Dataset(path_of_evaluation_data,transform=transform1)\n",
        "evaluationset_loader1 = torch.utils.data.DataLoader(evaluationset1, batch_size=BATCH_SIZE, shuffle=False)\n",
        "\n",
        "validset1 = Food_11_Dataset(path_of_validation_data,transform=transform1)\n",
        "validset_loader1 = torch.utils.data.DataLoader(validset1, batch_size=BATCH_SIZE, shuffle=False)\n",
        "\n",
        "\n",
        "trainset2 = Food_11_Dataset(path_of_training_data,transform=transform2)\n",
        "trainset_loader2 = torch.utils.data.DataLoader(trainset2, batch_size=BATCH_SIZE, shuffle=False)\n",
        "\n",
        "evaluationset2 = Food_11_Dataset(path_of_evaluation_data,transform=transform2)\n",
        "evaluationset_loader2 = torch.utils.data.DataLoader(evaluationset2, batch_size=BATCH_SIZE, shuffle=False)\n",
        "\n",
        "validset2 = Food_11_Dataset(path_of_validation_data,transform=transform2)\n",
        "validset_loader2 = torch.utils.data.DataLoader(validset2, batch_size=BATCH_SIZE, shuffle=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9Hi0X9JQ1CVn"
      },
      "outputs": [],
      "source": [
        "class MyEnsemble(torch.nn.Module):\n",
        "    def __init__(self, modelA,modelB,modelC,modelD):\n",
        "        super(MyEnsemble, self).__init__()\n",
        "        self.modelA = modelA\n",
        "        self.modelC = modelC\n",
        "        self.modelB = modelB\n",
        "        self.modelD = modelD\n",
        "        \n",
        "    def forward(self,x1,x2,x3,x4):\n",
        "        x1 = self.modelA(x1)\n",
        "        x2 = self.modelB(x2)\n",
        "        x3 = self.modelC(x3)\n",
        "        x4 = self.modelD(x4)\n",
        "        x =   x1+x2+x3+x4\n",
        "        x = torch.mul(x, (1.0/4.0))\n",
        "        return torch.softmax(x, dim=1)\n",
        "\n",
        "\n",
        "class MyEnsemble2(torch.nn.Module):\n",
        "    def __init__(self, modelA,modelB, modelC,modelD):\n",
        "        super(MyEnsemble2, self).__init__()\n",
        "        self.modelA = modelA\n",
        "        self.modelC = modelC\n",
        "        self.modelB = modelB\n",
        "        self.modelD = modelD\n",
        "        self.classifier = torch.nn.Linear(44, 11)\n",
        "        \n",
        "    def forward(self, x1,x2, x3,x4):\n",
        "        x1 = self.modelA(x1)\n",
        "        x3 = self.modelC(x3)\n",
        "        x2 = self.modelB(x2)\n",
        "        x4 = self.modelD(x4)\n",
        "        x = torch.cat((x1,x2, x3,x4), dim=1)\n",
        "        x = self.classifier(x)\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "PATH1 = \"/content/drive/MyDrive/Test105/path1.pth\"\n",
        "PATH2 = \"/content/drive/MyDrive/Test105/path2.pth\"\n",
        "PATH3 = \"/content/drive/MyDrive/Test105/path3.pth\"\n",
        "PATH4 = \"/content/drive/MyDrive/Test105/path4.pth\"\n",
        "PATH5 = \"/content/drive/MyDrive/Test105/path5.pth\"\n",
        "PATH6 = \"/content/drive/MyDrive/Test105/path6.pth\"\n",
        "PATH7 = \"/content/drive/MyDrive/Test105/path7.pth\"\n",
        "PATH8 = \"/content/drive/MyDrive/Test105/path8.pth\"\n",
        "\n",
        "PATH9 = \"/content/drive/MyDrive/Test105/path9.pth\"\n",
        "PATH10 = \"/content/drive/MyDrive/Test105/path10.pth\"\n",
        "PATH11 = \"/content/drive/MyDrive/Test105/path11.pth\"\n",
        "PATH12 = \"/content/drive/MyDrive/Test105/path12.pth\"\n",
        "PATH13 = \"/content/drive/MyDrive/Test105/path13.pth\"\n",
        "PATH14 = \"/content/drive/MyDrive/Test105/path14.pth\"\n",
        "\n",
        "\n",
        "\n",
        "#Traditional Ensemble\n",
        "modelA = torch.hub.load('pytorch/vision:v0.13.0', 'alexnet', pretrained=True)\n",
        "modelA.to(device)\n",
        "modelB = torch.hub.load('pytorch/vision:v0.13.0', 'resnet50', pretrained=True)\n",
        "modelB.to(device)\n",
        "modelC = torch.hub.load('pytorch/vision:v0.13.0', 'inception_v3', pretrained=True)\n",
        "modelC.aux_logits = False \n",
        "modelC.to(device)\n",
        "modelD = torch.hub.load('pytorch/vision:v0.13.0', 'vgg16', pretrained=True)\n",
        "modelD.to(device)\n",
        "modelA.classifier[6] = torch.nn.Linear(4096,11)\n",
        "modelB.fc = torch.nn.Linear(2048, 11)\n",
        "modelC.fc = torch.nn.Linear(2048, 11)\n",
        "modelD.classifier[6] = torch.nn.Linear(4096,11)\n",
        "model1 = MyEnsemble(modelA,modelB,modelC,modelD)\n",
        "model1.to(device)\n",
        "criterion1 = torch.nn.CrossEntropyLoss()\n",
        "optimizer1 = torch.optim.SGD(model1.parameters(), lr=LEARNING_RATE)\n",
        "\n",
        "\n",
        "if os.path.getsize(PATH1) > 0:\n",
        "  checkpoint = torch.load(PATH1, map_location=torch.device(device))\n",
        "  modelA.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "if os.path.getsize(PATH2) > 0:\n",
        "  checkpoint = torch.load(PATH2, map_location=torch.device(device))\n",
        "  modelC.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "if os.path.getsize(PATH3) > 0:\n",
        "  checkpoint = torch.load(PATH3, map_location=torch.device(device))\n",
        "  modelB.load_state_dict(checkpoint['model_state_dict'])\n",
        "if os.path.getsize(PATH4) > 0:\n",
        "  checkpoint = torch.load(PATH4, map_location=torch.device(device))\n",
        "  modelD.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "epoch1 = 0\n",
        "loss1 = 0.0\n",
        "best_acc1 = 0.0\n",
        "best_model_weights1 = []\n",
        "if os.path.getsize(PATH9) > 0:\n",
        "  checkpoint = torch.load(PATH9, map_location=torch.device(device))\n",
        "  model1.load_state_dict(checkpoint['model_state_dict'])\n",
        "  optimizer1.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "  epoch1 = checkpoint['epoch']\n",
        "  loss1 = checkpoint['loss']\n",
        "  best_acc1 = checkpoint['best_acc']\n",
        "  best_model_weights1 = copy.deepcopy(model1.state_dict())\n",
        "\n",
        "\n",
        "model1_info = {'train_loss' : [] , 'val_acc' : []}\n",
        "PATH1_acc = \"/content/drive/MyDrive/Test105/path1acc.pth\"\n",
        "if os.path.getsize(PATH1_acc) > 0:\n",
        "  model1_info = torch.load(PATH1_acc, map_location=torch.device(device))\n",
        "\n",
        "\n",
        "#Traditional Avg Ensemble\n",
        "modelA1 = torch.hub.load('pytorch/vision:v0.13.0', 'alexnet', pretrained=True)\n",
        "modelA1.to(device)\n",
        "modelB1 = torch.hub.load('pytorch/vision:v0.13.0', 'resnet50', pretrained=True)\n",
        "modelB1.to(device)\n",
        "modelC1 = torch.hub.load('pytorch/vision:v0.13.0', 'inception_v3', pretrained=True)\n",
        "modelC1.aux_logits = False \n",
        "modelC1.to(device)\n",
        "modelD1 = torch.hub.load('pytorch/vision:v0.13.0', 'vgg16', pretrained=True)\n",
        "modelD1.to(device)\n",
        "modelA1.classifier[6] = torch.nn.Linear(4096,11)\n",
        "modelB1.fc = torch.nn.Linear(2048, 11)\n",
        "modelC1.fc = torch.nn.Linear(2048, 11)\n",
        "modelD1.classifier[6] = torch.nn.Linear(4096,11)\n",
        "\n",
        "model2 = MyEnsemble(modelA1,modelB1,modelC1,modelD1)\n",
        "model2.to(device)\n",
        "criterion2 = torch.nn.CrossEntropyLoss()\n",
        "optimizer2 = torch.optim.SGD(model2.parameters(), lr=LEARNING_RATE)\n",
        "\n",
        "\n",
        "if os.path.getsize(PATH5) > 0:\n",
        "  checkpoint = torch.load(PATH5, map_location=torch.device(device))\n",
        "  modelA1.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "if os.path.getsize(PATH6) > 0:\n",
        "  checkpoint = torch.load(PATH6, map_location=torch.device(device))\n",
        "  modelC1.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "if os.path.getsize(PATH7) > 0:\n",
        "  checkpoint = torch.load(PATH7, map_location=torch.device(device))\n",
        "  modelB1.load_state_dict(checkpoint['model_state_dict'])\n",
        "if os.path.getsize(PATH8) > 0:\n",
        "  checkpoint = torch.load(PATH8, map_location=torch.device(device))\n",
        "  modelD1.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "epoch2 = 0\n",
        "loss2 = 0.0\n",
        "best_acc2 = 0.0\n",
        "best_model_weights2 = []\n",
        "if os.path.getsize(PATH10) > 0:\n",
        "  checkpoint = torch.load(PATH10, map_location=torch.device(device))\n",
        "  model2.load_state_dict(checkpoint['model_state_dict'])\n",
        "  optimizer2.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "  epoch2 = checkpoint['epoch']\n",
        "  loss2 = checkpoint['loss']\n",
        "  best_acc2 = checkpoint['best_acc']\n",
        "  best_model_weights2 = copy.deepcopy(model2.state_dict())\n",
        "\n",
        "\n",
        "model2_info = {'train_loss' : [] , 'val_acc' : []}\n",
        "PATH2_acc = \"/content/drive/MyDrive/Test105/path2acc.pth\"\n",
        "if os.path.getsize(PATH2_acc) > 0:\n",
        "  model2_info = torch.load(PATH2_acc, map_location=torch.device(device))\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "#Base Ensemble\n",
        "modelA2 = torch.hub.load('pytorch/vision:v0.13.0', 'alexnet', pretrained=True)\n",
        "modelA2.to(device)\n",
        "modelB2 = torch.hub.load('pytorch/vision:v0.13.0', 'resnet50', pretrained=True)\n",
        "modelB2.to(device)\n",
        "modelC2 = torch.hub.load('pytorch/vision:v0.13.0', 'inception_v3', pretrained=True)\n",
        "modelC2.aux_logits = False \n",
        "modelC2.to(device)\n",
        "modelD2 = torch.hub.load('pytorch/vision:v0.13.0', 'vgg16', pretrained=True)\n",
        "modelD2.to(device)\n",
        "modelA2.classifier[6] = torch.nn.Linear(4096,11)\n",
        "modelB2.fc = torch.nn.Linear(2048, 11)\n",
        "modelC2.fc = torch.nn.Linear(2048, 11)\n",
        "modelD2.classifier[6] = torch.nn.Linear(4096,11)\n",
        "model3 = MyEnsemble2(modelA2,modelB2,modelC2,modelD2)\n",
        "model3.to(device)\n",
        "criterion3 = torch.nn.CrossEntropyLoss()\n",
        "optimizer3 = torch.optim.SGD(model3.parameters(), lr=LEARNING_RATE)\n",
        "\n",
        "\n",
        "if os.path.getsize(PATH1) > 0:\n",
        "  checkpoint = torch.load(PATH1, map_location=torch.device(device))\n",
        "  modelA2.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "if os.path.getsize(PATH2) > 0:\n",
        "  checkpoint = torch.load(PATH2, map_location=torch.device(device))\n",
        "  modelC2.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "if os.path.getsize(PATH3) > 0:\n",
        "  checkpoint = torch.load(PATH3, map_location=torch.device(device))\n",
        "  modelB2.load_state_dict(checkpoint['model_state_dict'])\n",
        "if os.path.getsize(PATH4) > 0:\n",
        "  checkpoint = torch.load(PATH4, map_location=torch.device(device))\n",
        "  modelD2.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "epoch3 = 0\n",
        "loss3 = 0.0\n",
        "best_acc3 = 0.0\n",
        "best_model_weights3 = []\n",
        "\n",
        "if os.path.getsize(PATH11) > 0:\n",
        "  checkpoint = torch.load(PATH11, map_location=torch.device(device))\n",
        "  model3.load_state_dict(checkpoint['model_state_dict'])\n",
        "  optimizer3.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "  epoch3 = checkpoint['epoch']\n",
        "  loss3 = checkpoint['loss']\n",
        "  best_acc3 = checkpoint['best_acc']\n",
        "  best_model_weights3 = copy.deepcopy(model3.state_dict())\n",
        "\n",
        "\n",
        "model3_info = {'train_loss' : [] , 'val_acc' : []}\n",
        "PATH3_acc = \"/content/drive/MyDrive/Test105/path3acc.pth\"\n",
        "if os.path.getsize(PATH3_acc) > 0:\n",
        "  model3_info = torch.load(PATH3_acc, map_location=torch.device(device))\n",
        "\n",
        "\n",
        "\n",
        "#Avg Ensemble\n",
        "modelA3 = torch.hub.load('pytorch/vision:v0.13.0', 'alexnet', pretrained=True)\n",
        "modelA3.to(device)\n",
        "modelB3 = torch.hub.load('pytorch/vision:v0.13.0', 'resnet50', pretrained=True)\n",
        "modelB3.to(device)\n",
        "modelC3 = torch.hub.load('pytorch/vision:v0.13.0', 'inception_v3', pretrained=True)\n",
        "modelC3.aux_logits = False \n",
        "modelC3.to(device)\n",
        "modelD3 = torch.hub.load('pytorch/vision:v0.13.0', 'vgg16', pretrained=True)\n",
        "modelD3.to(device)\n",
        "modelA3.classifier[6] = torch.nn.Linear(4096,11)\n",
        "modelB3.fc = torch.nn.Linear(2048, 11)\n",
        "modelC3.fc = torch.nn.Linear(2048, 11)\n",
        "modelD3.classifier[6] = torch.nn.Linear(4096,11)\n",
        "\n",
        "model4 = MyEnsemble2(modelA3,modelB3,modelC3,modelD3)\n",
        "model4.to(device)\n",
        "criterion4 = torch.nn.CrossEntropyLoss()\n",
        "optimizer4 = torch.optim.SGD(model4.parameters(), lr=LEARNING_RATE)\n",
        "\n",
        "\n",
        "if os.path.getsize(PATH5) > 0:\n",
        "  checkpoint = torch.load(PATH5, map_location=torch.device(device))\n",
        "  modelA3.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "if os.path.getsize(PATH6) > 0:\n",
        "  checkpoint = torch.load(PATH6, map_location=torch.device(device))\n",
        "  modelC3.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "if os.path.getsize(PATH7) > 0:\n",
        "  checkpoint = torch.load(PATH7, map_location=torch.device(device))\n",
        "  modelB3.load_state_dict(checkpoint['model_state_dict'])\n",
        "if os.path.getsize(PATH8) > 0:\n",
        "  checkpoint = torch.load(PATH8, map_location=torch.device(device))\n",
        "  modelD3.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "epoch4 = 0\n",
        "loss4 = 0.0\n",
        "best_acc4 = 0.0\n",
        "best_model_weights4 = []\n",
        "\n",
        "if os.path.getsize(PATH12) > 0:\n",
        "  checkpoint = torch.load(PATH12, map_location=torch.device(device))\n",
        "  model4.load_state_dict(checkpoint['model_state_dict'])\n",
        "  optimizer4.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "  epoch4 = checkpoint['epoch']\n",
        "  loss4 = checkpoint['loss']\n",
        "  best_acc4 = checkpoint['best_acc']\n",
        "  best_model_weights4 = copy.deepcopy(model4.state_dict())\n",
        "\n",
        "\n",
        "model4_info = {'train_loss' : [] , 'val_acc' : []}\n",
        "PATH4_acc = \"/content/drive/MyDrive/Test105/path4acc.pth\"\n",
        "if os.path.getsize(PATH4_acc) > 0:\n",
        "  model4_info = torch.load(PATH4_acc, map_location=torch.device(device))\n",
        "\n",
        "\n",
        "#Base Ensemble Avg\n",
        "modelA4 = torch.hub.load('pytorch/vision:v0.13.0', 'alexnet', pretrained=True)\n",
        "modelA4.to(device)\n",
        "modelC4 = torch.hub.load('pytorch/vision:v0.13.0', 'inception_v3', pretrained=True)\n",
        "modelC4.aux_logits = False \n",
        "modelC4.to(device)\n",
        "modelB4 = torch.hub.load('pytorch/vision:v0.13.0', 'resnet50', pretrained=True)\n",
        "modelB4.to(device)\n",
        "modelD4 = torch.hub.load('pytorch/vision:v0.13.0', 'vgg16', pretrained=True)\n",
        "modelD4.to(device)\n",
        "modelA4.classifier[6] = torch.nn.Linear(4096,11)\n",
        "modelB4.fc = torch.nn.Linear(2048, 11)\n",
        "modelC4.fc = torch.nn.Linear(2048, 11)\n",
        "modelD4.classifier[6] = torch.nn.Linear(4096,11)\n",
        "\n",
        "model5 = MyEnsemble2(modelA4,modelB4,modelC4,modelD4)\n",
        "model5.to(device)\n",
        "criterion5 = torch.nn.CrossEntropyLoss()\n",
        "optimizer5 = torch.optim.SGD(model5.parameters(), lr=LEARNING_RATE)\n",
        "if os.path.getsize(PATH1) > 0:\n",
        "  checkpoint = torch.load(PATH1, map_location=torch.device(device))\n",
        "  modelA4.load_state_dict(checkpoint['model_state_dict'])\n",
        "if os.path.getsize(PATH2) > 0:\n",
        "  checkpoint = torch.load(PATH2, map_location=torch.device(device))\n",
        "  modelC4.load_state_dict(checkpoint['model_state_dict'])\n",
        "if os.path.getsize(PATH3) > 0:\n",
        "  checkpoint = torch.load(PATH3, map_location=torch.device(device))\n",
        "  modelB4.load_state_dict(checkpoint['model_state_dict'])\n",
        "if os.path.getsize(PATH4) > 0:\n",
        "  checkpoint = torch.load(PATH4, map_location=torch.device(device))\n",
        "  modelD4.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "epoch5 = 0\n",
        "loss5 = 0.0\n",
        "best_acc5 = 0.0\n",
        "best_model_weights5 = []\n",
        "if os.path.getsize(PATH13) > 0:\n",
        "  checkpoint = torch.load(PATH13, map_location=torch.device(device))\n",
        "  model5.load_state_dict(checkpoint['model_state_dict'])\n",
        "  optimizer5.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "  epoch5 = checkpoint['epoch']\n",
        "  loss5 = checkpoint['loss']\n",
        "  best_acc5 = checkpoint['best_acc']\n",
        "  best_model_weights5 = copy.deepcopy(model5.state_dict())\n",
        "\n",
        "\n",
        "model5_info = {'train_loss' : [] , 'val_acc' : []}\n",
        "PATH5_acc = \"/content/drive/MyDrive/Test105/path5acc.pth\"\n",
        "if os.path.getsize(PATH5_acc) > 0:\n",
        "  model5_info = torch.load(PATH5_acc, map_location=torch.device(device))\n",
        "\n",
        "AVG_PATH1 = \"/content/drive/MyDrive/Test105/avgpath1.pth\"\n",
        "avg_model_weights1 = []\n",
        "if os.path.getsize(AVG_PATH1) > 0:\n",
        "  checkpoint = torch.load(AVG_PATH1)\n",
        "  avg_model_weights1 = checkpoint['model_state_dict']\n",
        "\n",
        "prev_model1 = []\n",
        "if len(avg_model_weights1) != 0:\n",
        "  prev_model1 = avg_model_weights1\n",
        "\n",
        "\n",
        "#Avg Ensemble Avg\n",
        "modelA5 = torch.hub.load('pytorch/vision:v0.13.0', 'alexnet', pretrained=True)\n",
        "modelA5.to(device)\n",
        "modelC5 = torch.hub.load('pytorch/vision:v0.13.0', 'inception_v3', pretrained=True)\n",
        "modelC5.aux_logits = False \n",
        "modelC5.to(device)\n",
        "modelB5 = torch.hub.load('pytorch/vision:v0.13.0', 'resnet50', pretrained=True)\n",
        "modelB5.to(device)\n",
        "modelD5 = torch.hub.load('pytorch/vision:v0.13.0', 'vgg16', pretrained=True)\n",
        "modelD5.to(device)\n",
        "modelA5.classifier[6] = torch.nn.Linear(4096,11)\n",
        "modelB5.fc = torch.nn.Linear(2048, 11)\n",
        "modelC5.fc = torch.nn.Linear(2048, 11)\n",
        "modelD5.classifier[6] = torch.nn.Linear(4096,11)\n",
        "\n",
        "model6 = MyEnsemble2(modelA5,modelB5,modelC5,modelD5)\n",
        "model6.to(device)\n",
        "criterion6 = torch.nn.CrossEntropyLoss()\n",
        "optimizer6 = torch.optim.SGD(model6.parameters(), lr=LEARNING_RATE)\n",
        "if os.path.getsize(PATH5) > 0:\n",
        "  checkpoint = torch.load(PATH5, map_location=torch.device(device))\n",
        "  modelA5.load_state_dict(checkpoint['model_state_dict'])\n",
        "if os.path.getsize(PATH6) > 0:\n",
        "  checkpoint = torch.load(PATH6, map_location=torch.device(device))\n",
        "  modelC5.load_state_dict(checkpoint['model_state_dict'])\n",
        "if os.path.getsize(PATH7) > 0:\n",
        "  checkpoint = torch.load(PATH7, map_location=torch.device(device))\n",
        "  modelB5.load_state_dict(checkpoint['model_state_dict'])\n",
        "if os.path.getsize(PATH8) > 0:\n",
        "  checkpoint = torch.load(PATH8, map_location=torch.device(device))\n",
        "  modelD5.load_state_dict(checkpoint['model_state_dict'])\n",
        "\n",
        "epoch6 = 0\n",
        "loss6 = 0.0\n",
        "best_acc6 = 0.0\n",
        "best_model_weights6 = []\n",
        "\n",
        "if os.path.getsize(PATH14) > 0:\n",
        "  checkpoint = torch.load(PATH14, map_location=torch.device(device))\n",
        "  model6.load_state_dict(checkpoint['model_state_dict'])\n",
        "  optimizer6.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "  epoch6 = checkpoint['epoch']\n",
        "  loss6 = checkpoint['loss']\n",
        "  best_acc6 = checkpoint['best_acc']\n",
        "  best_model_weights6 = copy.deepcopy(model6.state_dict())\n",
        "\n",
        "\n",
        "model6_info = {'train_loss' : [] , 'val_acc' : []}\n",
        "PATH6_acc = \"/content/drive/MyDrive/Test105/path6acc.pth\"\n",
        "if os.path.getsize(PATH6_acc) > 0:\n",
        "  model6_info = torch.load(PATH6_acc, map_location=torch.device(device))\n",
        "\n",
        "AVG_PATH2 = \"/content/drive/MyDrive/Test105/avgpath2.pth\"\n",
        "avg_model_weights2 = []\n",
        "if os.path.getsize(AVG_PATH2) > 0:\n",
        "  checkpoint = torch.load(AVG_PATH2)\n",
        "  avg_model_weights2 = checkpoint['model_state_dict']\n",
        "\n",
        "prev_model2 = []\n",
        "if len(avg_model_weights2) != 0:\n",
        "  prev_model2 = avg_model_weights2\n",
        "\n",
        "start_time = time.time()\n",
        "new_time1 = time.time()\n",
        "total_step = len(trainset_loader1)"
      ],
      "metadata": {
        "id": "enIhwLC4cPCw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "t594VDiW1SdT"
      },
      "outputs": [],
      "source": [
        "for epoch in range(NUM_EPOCHS):\n",
        "\n",
        "#NOW NODEL 1\n",
        "\n",
        "    i = 0\n",
        "    new_time1 = time.time()\n",
        "    start_time = time.time()\n",
        "    model1.train()\n",
        "    model2.train()\n",
        "    model3.train()\n",
        "    model4.train()\n",
        "    model5.train()\n",
        "    model6.train()\n",
        "    print('Epoch [{}/{}] --------- '.format(epoch+1, NUM_EPOCHS))\n",
        "    for a,b in zip(trainset_loader1,trainset_loader2):\n",
        "        optimizer1.zero_grad()\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs1 = model1(images,images,v3_images,images)\n",
        "        loss1 = criterion1(outputs1, labels)\n",
        "        loss1.backward()\n",
        "        optimizer1.step()\n",
        "    model1_info['train_loss'].append(loss1.item())\n",
        "\n",
        "    print ('Model 1  Step [{}/{}], Loss: {:.4f}' \n",
        "                   .format(i+1, total_step, loss1.item()))\n",
        "  \n",
        "    # Validation\n",
        "\n",
        "    model1.eval()\n",
        "    with torch.set_grad_enabled(False):\n",
        "        correct = 0\n",
        "        total = 0\n",
        "        for a,b in zip(validset_loader1,validset_loader2):\n",
        "            images,labels = a\n",
        "            v3_images,v3_labels = b\n",
        "            images = images.to(device)\n",
        "            labels = labels.to(device)\n",
        "            v3_images = v3_images.to(device)\n",
        "            outputs =  model1(images,images,v3_images,images)\n",
        "            _, predicted = torch.max(outputs.data, 1)\n",
        "            total += labels.size(0)\n",
        "            correct += (predicted == labels).sum().item()\n",
        "            del images, labels, v3_images, v3_labels, outputs\n",
        "    \n",
        "        print('Accuracy of the network on the {} valid set images: {} %'.format(total, 100 * correct / total)) \n",
        "        model1_info['val_acc'].append((100 * correct / total))\n",
        "        torch.save(model1_info,PATH1_acc)\n",
        "        if (100 * correct / total) > best_acc1 : \n",
        "            best_acc1 = (100 * correct / total)\n",
        "            best_model_weights1 = copy.deepcopy(model1.state_dict())\n",
        "            torch.save({\n",
        "            'epoch': epoch,\n",
        "            'model_state_dict': model1.state_dict(),\n",
        "            'optimizer_state_dict': optimizer1.state_dict(),\n",
        "            'loss': loss1,\n",
        "            'best_acc' : best_acc1,\n",
        "            },PATH9)\n",
        "\n",
        "    for a,b in zip(trainset_loader1,trainset_loader2):\n",
        "        optimizer1.zero_grad()\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs1 = model2(images,images,v3_images,images)\n",
        "        loss2 = criterion2(outputs1, labels)\n",
        "        loss2.backward()\n",
        "        optimizer2.step()\n",
        "\n",
        "    model2_info['train_loss'].append(loss2.item())\n",
        "    print ('Model 2  Step [{}/{}], Loss: {:.4f}' \n",
        "                   .format(i+1, total_step, loss2.item()))\n",
        "  \n",
        "    # Validation\n",
        "\n",
        "    model2.eval()\n",
        "    with torch.set_grad_enabled(False):\n",
        "        correct = 0\n",
        "        total = 0\n",
        "        for a,b in zip(validset_loader1,validset_loader2):\n",
        "            images,labels = a\n",
        "            v3_images,v3_labels = b\n",
        "            images = images.to(device)\n",
        "            labels = labels.to(device)\n",
        "            v3_images = v3_images.to(device)\n",
        "            outputs =  model2(images,images,v3_images,images)\n",
        "            _, predicted = torch.max(outputs.data, 1)\n",
        "            total += labels.size(0)\n",
        "            correct += (predicted == labels).sum().item()\n",
        "            del images, labels, v3_images, v3_labels, outputs\n",
        "    \n",
        "        print('Accuracy of the network on the {} valid set images: {} %'.format(total, 100 * correct / total)) \n",
        "        model2_info['val_acc'].append((100 * correct / total))\n",
        "        torch.save(model2_info,PATH2_acc)\n",
        "        if (100 * correct / total) > best_acc2 : \n",
        "            best_acc2 = (100 * correct / total)\n",
        "            best_model_weights2 = copy.deepcopy(model2.state_dict())\n",
        "            torch.save({\n",
        "            'epoch': epoch,\n",
        "            'model_state_dict': model2.state_dict(),\n",
        "            'optimizer_state_dict': optimizer2.state_dict(),\n",
        "            'loss': loss2,\n",
        "            'best_acc' : best_acc2,\n",
        "            },PATH10)\n",
        "\n",
        "    for a,b in zip(trainset_loader1,trainset_loader2):\n",
        "        optimizer1.zero_grad()\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs1 = model3(images,images,v3_images,images)\n",
        "        loss3 = criterion3(outputs1, labels)\n",
        "        loss3.backward()\n",
        "        optimizer3.step()\n",
        "\n",
        "    model3_info['train_loss'].append(loss3.item())\n",
        "    print ('Model 3  Step [{}/{}], Loss: {:.4f}' \n",
        "                   .format(i+1, total_step, loss3.item()))\n",
        "  \n",
        "    # Validation\n",
        "\n",
        "    model3.eval()\n",
        "    with torch.set_grad_enabled(False):\n",
        "        correct = 0\n",
        "        total = 0\n",
        "        for a,b in zip(validset_loader1,validset_loader2):\n",
        "            images,labels = a\n",
        "            v3_images,v3_labels = b\n",
        "            images = images.to(device)\n",
        "            labels = labels.to(device)\n",
        "            v3_images = v3_images.to(device)\n",
        "            outputs =  model3(images,images,v3_images,images)\n",
        "            _, predicted = torch.max(outputs.data, 1)\n",
        "            total += labels.size(0)\n",
        "            correct += (predicted == labels).sum().item()\n",
        "            del images, labels, v3_images, v3_labels, outputs\n",
        "    \n",
        "        print('Accuracy of the network on the {} valid set images: {} %'.format(total, 100 * correct / total)) \n",
        "        model3_info['val_acc'].append((100 * correct / total))\n",
        "        torch.save(model3_info,PATH3_acc)\n",
        "        if (100 * correct / total) > best_acc3 : \n",
        "            best_acc3 = (100 * correct / total)\n",
        "            best_model_weights3 = copy.deepcopy(model3.state_dict())\n",
        "            torch.save({\n",
        "            'epoch': epoch,\n",
        "            'model_state_dict': model3.state_dict(),\n",
        "            'optimizer_state_dict': optimizer3.state_dict(),\n",
        "            'loss': loss3,\n",
        "            'best_acc' : best_acc3,\n",
        "            },PATH11)\n",
        "\n",
        "    for a,b in zip(trainset_loader1,trainset_loader2):\n",
        "        optimizer1.zero_grad()\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs1 = model4(images,images,v3_images,images)\n",
        "        loss4 = criterion4(outputs1, labels)\n",
        "        loss4.backward()\n",
        "        optimizer4.step()\n",
        "\n",
        "    model4_info['train_loss'].append(loss4.item())\n",
        "    print ('Model 4  Step [{}/{}], Loss: {:.4f}' \n",
        "                    .format(i+1, total_step, loss4.item()))\n",
        "\n",
        "    # Validation\n",
        "\n",
        "    model4.eval()\n",
        "    with torch.set_grad_enabled(False):\n",
        "        correct = 0\n",
        "        total = 0\n",
        "        for a,b in zip(validset_loader1,validset_loader2):\n",
        "            images,labels = a\n",
        "            v3_images,v3_labels = b\n",
        "            images = images.to(device)\n",
        "            labels = labels.to(device)\n",
        "            v3_images = v3_images.to(device)\n",
        "            outputs =  model4(images,images,v3_images,images)\n",
        "            _, predicted = torch.max(outputs.data, 1)\n",
        "            total += labels.size(0)\n",
        "            correct += (predicted == labels).sum().item()\n",
        "            del images, labels, v3_images, v3_labels, outputs\n",
        "\n",
        "        print('Accuracy of the network on the {} valid set images: {} %'.format(total, 100 * correct / total)) \n",
        "        model4_info['val_acc'].append((100 * correct / total))\n",
        "        torch.save(model4_info,PATH4_acc)\n",
        "        if (100 * correct / total) > best_acc4 : \n",
        "            best_acc4 = (100 * correct / total)\n",
        "            best_model_weights4 = copy.deepcopy(model4.state_dict())\n",
        "            torch.save({\n",
        "            'epoch': epoch,\n",
        "            'model_state_dict': model4.state_dict(),\n",
        "            'optimizer_state_dict': optimizer4.state_dict(),\n",
        "            'loss': loss4,\n",
        "            'best_acc' : best_acc4,\n",
        "            },PATH12)\n",
        "\n",
        "\n",
        "\n",
        "    for a,b in zip(trainset_loader1,trainset_loader2):\n",
        "        optimizer1.zero_grad()\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs1 = model5(images,images,v3_images,images)\n",
        "        loss5 = criterion5(outputs1, labels)\n",
        "        loss5.backward()\n",
        "        optimizer5.step()\n",
        "\n",
        "    model5_info['train_loss'].append(loss5.item())\n",
        "    print ('Model 5  Step [{}/{}], Loss: {:.5f}' \n",
        "                    .format(i+1, total_step, loss5.item()))\n",
        "\n",
        "    if epoch+1 == 5:\n",
        "\n",
        "        prev_model1 = best_model_weights5\n",
        "\n",
        "        \n",
        "    elif epoch+1 > 5:\n",
        "\n",
        "        temp_model = copy.deepcopy(model5.state_dict())\n",
        "        for key in prev_model1:\n",
        "            prev_model1[key] = (prev_model1[key] + temp_model[key]) / 2.0\n",
        "        model5.load_state_dict(prev_model1)\n",
        "        torch.save({\n",
        "                'model_state_dict': model5.state_dict(),\n",
        "                },AVG_PATH1) \n",
        "\n",
        "    # Validation\n",
        "\n",
        "    model5.eval()\n",
        "    with torch.set_grad_enabled(False):\n",
        "        correct = 0\n",
        "        total = 0\n",
        "        for a,b in zip(validset_loader1,validset_loader2):\n",
        "            images,labels = a\n",
        "            v3_images,v3_labels = b\n",
        "            images = images.to(device)\n",
        "            labels = labels.to(device)\n",
        "            v3_images = v3_images.to(device)\n",
        "            outputs =  model5(images,images,v3_images,images)\n",
        "            _, predicted = torch.max(outputs.data, 1)\n",
        "            total += labels.size(0)\n",
        "            correct += (predicted == labels).sum().item()\n",
        "            del images, labels, v3_images, v3_labels, outputs\n",
        "\n",
        "        print('Accuracy of the network on the {} valid set images: {} %'.format(total, 100 * correct / total)) \n",
        "        model5_info['val_acc'].append((100 * correct / total))\n",
        "        torch.save(model5_info,PATH5_acc)\n",
        "        if (100 * correct / total) > best_acc5 : \n",
        "            best_acc5 = (100 * correct / total)\n",
        "            best_model_weights5 = copy.deepcopy(model5.state_dict())\n",
        "            torch.save({\n",
        "            'epoch': epoch,\n",
        "            'model_state_dict': model5.state_dict(),\n",
        "            'optimizer_state_dict': optimizer5.state_dict(),\n",
        "            'loss': loss5,\n",
        "            'best_acc' : best_acc5,\n",
        "            },PATH13)\n",
        "\n",
        "    for a,b in zip(trainset_loader1,trainset_loader2):\n",
        "        optimizer1.zero_grad()\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs1 = model6(images,images,v3_images,images)\n",
        "        loss6 = criterion6(outputs1, labels)\n",
        "        loss6.backward()\n",
        "        optimizer6.step()\n",
        "\n",
        "    model6_info['train_loss'].append(loss6.item())\n",
        "    print ('Model 6 Step [{}/{}], Loss: {:.5f}' \n",
        "                    .format(i+1, total_step, loss6.item()))\n",
        "\n",
        "    if epoch+1 == 5:\n",
        "\n",
        "        prev_model2 = best_model_weights6\n",
        "\n",
        "        \n",
        "    elif epoch+1 > 5:\n",
        "\n",
        "        temp_model = copy.deepcopy(model6.state_dict())\n",
        "        for key in prev_model2:\n",
        "            prev_model2[key] = (prev_model2[key] + temp_model[key]) / 2.0\n",
        "        model6.load_state_dict(prev_model2)\n",
        "        torch.save({\n",
        "                'model_state_dict': model6.state_dict(),\n",
        "                },AVG_PATH2) \n",
        "\n",
        "    # Validation\n",
        "\n",
        "    model6.eval()\n",
        "    with torch.set_grad_enabled(False):\n",
        "        correct = 0\n",
        "        total = 0\n",
        "        for a,b in zip(validset_loader1,validset_loader2):\n",
        "            images,labels = a\n",
        "            v3_images,v3_labels = b\n",
        "            images = images.to(device)\n",
        "            labels = labels.to(device)\n",
        "            v3_images = v3_images.to(device)\n",
        "            outputs =  model6(images,images,v3_images,images)\n",
        "            _, predicted = torch.max(outputs.data, 1)\n",
        "            total += labels.size(0)\n",
        "            correct += (predicted == labels).sum().item()\n",
        "            del images, labels, v3_images, v3_labels, outputs\n",
        "\n",
        "        print('Accuracy of the network on the {} valid set images: {} %'.format(total, 100 * correct / total)) \n",
        "        model6_info['val_acc'].append((100 * correct / total))\n",
        "        torch.save(model6_info,PATH6_acc)\n",
        "        if (100 * correct / total) > best_acc6 : \n",
        "            best_acc6 = (100 * correct / total)\n",
        "            best_model_weights6 = copy.deepcopy(model6.state_dict())\n",
        "            torch.save({\n",
        "            'epoch': epoch,\n",
        "            'model_state_dict': model6.state_dict(),\n",
        "            'optimizer_state_dict': optimizer6.state_dict(),\n",
        "            'loss': loss6,\n",
        "            'best_acc' : best_acc6,\n",
        "            },PATH14)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "model1.load_state_dict(best_model_weights1)\n",
        "model1.eval()\n",
        "with torch.set_grad_enabled(False):\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    for a,b in zip(validset_loader1,validset_loader2):\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs =  model1(images,images,v3_images,images)\n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "        del images, labels, v3_images, v3_labels, outputs\n",
        "\n",
        "    print('Accuracy of the model 1 on the {} validation images: {} %'.format(total, 100 * correct / total)) \n",
        "\n",
        "model1.eval()\n",
        "with torch.set_grad_enabled(False):\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    for a,b in zip(evaluationset_loader1,evaluationset_loader2):\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs =  model1(images,images,v3_images,images)\n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "        del images, labels, v3_images, v3_labels, outputs\n",
        "\n",
        "    print('Accuracy of the model 1 on the {} evaluation images: {} %'.format(total, 100 * correct / total))\n",
        "\n",
        "model2.load_state_dict(best_model_weights2)\n",
        "model2.eval()\n",
        "with torch.set_grad_enabled(False):\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    for a,b in zip(validset_loader1,validset_loader2):\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs =  model2(images,images,v3_images,images)\n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "        del images, labels, v3_images, v3_labels, outputs\n",
        "\n",
        "    print('Accuracy of the model 2 on the {} validation images: {} %'.format(total, 100 * correct / total)) \n",
        "\n",
        "model2.eval()\n",
        "with torch.set_grad_enabled(False):\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    for a,b in zip(evaluationset_loader1,evaluationset_loader2):\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs =  model2(images,images,v3_images,images)\n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "        del images, labels, v3_images, v3_labels, outputs\n",
        "\n",
        "    print('Accuracy of the model 2 on the {} evaluation images: {} %'.format(total, 100 * correct / total))\n",
        "\n",
        "model3.load_state_dict(best_model_weights3)\n",
        "model3.eval()\n",
        "with torch.set_grad_enabled(False):\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    for a,b in zip(validset_loader1,validset_loader2):\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs =  model3(images,images,v3_images,images)\n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "        del images, labels, v3_images, v3_labels, outputs\n",
        "\n",
        "    print('Accuracy of the model 3 on the {} validation images: {} %'.format(total, 100 * correct / total)) \n",
        "\n",
        "model3.eval()\n",
        "with torch.set_grad_enabled(False):\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    for a,b in zip(evaluationset_loader1,evaluationset_loader2):\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs =  model3(images,images,v3_images,images)\n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "        del images, labels, v3_images, v3_labels, outputs\n",
        "\n",
        "    print('Accuracy of the model 3 on the {} evaluation images: {} %'.format(total, 100 * correct / total))\n",
        "\n",
        "model4.load_state_dict(best_model_weights4)\n",
        "model4.eval()\n",
        "with torch.set_grad_enabled(False):\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    for a,b in zip(validset_loader1,validset_loader2):\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs =  model4(images,images,v3_images,images)\n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "        del images, labels, v3_images, v3_labels, outputs\n",
        "\n",
        "    print('Accuracy of the model 4 on the {} validation images: {} %'.format(total, 100 * correct / total)) \n",
        "\n",
        "model4.eval()\n",
        "with torch.set_grad_enabled(False):\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    for a,b in zip(evaluationset_loader1,evaluationset_loader2):\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs =  model4(images,images,v3_images,images)\n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "        del images, labels, v3_images, v3_labels, outputs\n",
        "\n",
        "    print('Accuracy of the model 4 on the {} evaluation images: {} %'.format(total, 100 * correct / total))\n",
        "\n",
        "model5.load_state_dict(best_model_weights5)\n",
        "model5.eval()\n",
        "with torch.set_grad_enabled(False):\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    for a,b in zip(validset_loader1,validset_loader2):\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs =  model5(images,images,v3_images,images)\n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "        del images, labels, v3_images, v3_labels, outputs\n",
        "\n",
        "    print('Accuracy of the model 5 on the {} validation images: {} %'.format(total, 100 * correct / total)) \n",
        "\n",
        "model5.eval()\n",
        "with torch.set_grad_enabled(False):\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    for a,b in zip(evaluationset_loader1,evaluationset_loader2):\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs =  model5(images,images,v3_images,images)\n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "        del images, labels, v3_images, v3_labels, outputs\n",
        "\n",
        "    print('Accuracy of the model 5 on the {} evaluation images: {} %'.format(total, 100 * correct / total))\n",
        "\n",
        "model6.load_state_dict(best_model_weights6)\n",
        "model6.eval()\n",
        "with torch.set_grad_enabled(False):\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    for a,b in zip(validset_loader1,validset_loader2):\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs =  model6(images,images,v3_images,images)\n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "        del images, labels, v3_images, v3_labels, outputs\n",
        "\n",
        "    print('Accuracy of the model 6 on the {} validation images: {} %'.format(total, 100 * correct / total)) \n",
        "\n",
        "model6.eval()\n",
        "with torch.set_grad_enabled(False):\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    for a,b in zip(evaluationset_loader1,evaluationset_loader2):\n",
        "        images,labels = a\n",
        "        v3_images,v3_labels = b\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        v3_images = v3_images.to(device)\n",
        "        outputs =  model6(images,images,v3_images,images)\n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "        del images, labels, v3_images, v3_labels, outputs\n",
        "\n",
        "    print('Accuracy of the model 6 on the {} evaluation images: {} %'.format(total, 100 * correct / total))\n",
        "plt.figure(figsize=(10,5))\n",
        "plt.title(\"Training loss and Validation Accuracy of model 1\")\n",
        "plt.plot(model1_info['val_acc'],label=\"val\")\n",
        "plt.plot(model1_info['train_loss'],label=\"train\")\n",
        "plt.xlabel(\"iterations\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.show()\n",
        "\n",
        "plt.figure(figsize=(10,5))\n",
        "plt.title(\"Training loss and Validation Accuracy of model 2\")\n",
        "plt.plot(model2_info['val_acc'],label=\"val\")\n",
        "plt.plot(model2_info['train_loss'],label=\"train\")\n",
        "plt.xlabel(\"iterations\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.show()\n",
        "\n",
        "plt.figure(figsize=(10,5))\n",
        "plt.title(\"Training loss and Validation Accuracy of model 3\")\n",
        "plt.plot(model3_info['val_acc'],label=\"val\")\n",
        "plt.plot(model3_info['train_loss'],label=\"train\")\n",
        "plt.xlabel(\"iterations\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.show()\n",
        "\n",
        "\n",
        "plt.figure(figsize=(10,5))\n",
        "plt.title(\"Training loss and Validation Accuracy of model 4\")\n",
        "plt.plot(model4_info['val_acc'],label=\"val\")\n",
        "plt.plot(model4_info['train_loss'],label=\"train\")\n",
        "plt.xlabel(\"iterations\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.show()\n",
        "\n",
        "plt.figure(figsize=(10,5))\n",
        "plt.title(\"Training loss and Validation Accuracy of model 4\")\n",
        "plt.plot(model5_info['val_acc'],label=\"val\")\n",
        "plt.plot(model5_info['train_loss'],label=\"train\")\n",
        "plt.xlabel(\"iterations\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.show()\n",
        "\n",
        "\n",
        "plt.figure(figsize=(10,5))\n",
        "plt.title(\"Training loss and Validation Accuracy of model 4\")\n",
        "plt.plot(model6_info['val_acc'],label=\"val\")\n",
        "plt.plot(model6_info['train_loss'],label=\"train\")\n",
        "plt.xlabel(\"iterations\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.show()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": [],
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.8 (tags/v3.10.8:aaaf517, Oct 11 2022, 16:50:30) [MSC v.1933 64 bit (AMD64)]"
    },
    "vscode": {
      "interpreter": {
        "hash": "99f3cd6e142120424ee9b340f0a72fd1732cf789e0a499e2c90251ee424f3513"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}