Train_Model_DenseNet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import torch\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as datasets\n",
    "import sklearn.metrics as metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply the needed transformation to the data, as required by the Pytorch framework. <br/>\n",
    "Then, upload the data into DataLoader objects, which will be given as inputs to the model.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train size: 151293 \n",
      "Validation size: 35876 \n",
      "Test size: 35659\n"
     ]
    }
   ],
   "source": [
    "# Source: \n",
    "# https://www.learnopencv.com/image-classification-using-transfer-learning-in-pytorch/\n",
    "\n",
    "# Applying Transforms to the Data. These are needed for compatibility with the pytorch implementation of the model \n",
    "image_transforms = {\n",
    "    'train': transforms.Compose([\n",
    "        transforms.CenterCrop(size=224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406],\n",
    "                             [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    'valid': transforms.Compose([\n",
    "        transforms.CenterCrop(size=224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406],\n",
    "                             [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    'test': transforms.Compose([\n",
    "        transforms.CenterCrop(size=224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406],\n",
    "                             [0.229, 0.224, 0.225])\n",
    "    ])\n",
    "}\n",
    "\n",
    "\n",
    "# Set train and valid directory paths\n",
    "train_directory = '/home/advo/PycharmProjects/ML_ND3_CapstoneProject/Dataset_small/patches/train'\n",
    "valid_directory = '/home/advo/PycharmProjects/ML_ND3_CapstoneProject/Dataset_small/patches/valid'\n",
    "test_directory = '/home/advo/PycharmProjects/ML_ND3_CapstoneProject/Dataset_small/patches/test'\n",
    " \n",
    "# Batch size\n",
    "bs = 64\n",
    " \n",
    "# Number of classes\n",
    "num_classes = 2\n",
    " \n",
    "# Load Data from folders\n",
    "data = {\n",
    "    'train': datasets.ImageFolder(root=train_directory, transform=image_transforms['train']),\n",
    "    'valid': datasets.ImageFolder(root=valid_directory, transform=image_transforms['valid']),\n",
    "    'test': datasets.ImageFolder(root=test_directory, transform=image_transforms['test'])\n",
    "}\n",
    " \n",
    "# Size of Data, to be used for calculating Average Loss and Accuracy\n",
    "train_data_size = len(data['train'])\n",
    "valid_data_size = len(data['valid'])\n",
    "test_data_size = len(data['test'])\n",
    " \n",
    "# Create iterators for the Data loaded using DataLoader module\n",
    "train_data = torch.utils.data.DataLoader(data['train'], batch_size=bs, shuffle=True)\n",
    "valid_data = torch.utils.data.DataLoader(data['valid'], batch_size=bs, shuffle=True)\n",
    "test_data = torch.utils.data.DataLoader(data['test'], batch_size=bs, shuffle=True)\n",
    " \n",
    "# Print the train, validation and test set data sizes\n",
    "print(f\"Train size: {train_data_size} \\nValidation size: {valid_data_size} \\nTest size: {test_data_size}\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pretrained AlexNet Model\n",
    "alexnet = torchvision.models.alexnet(pretrained=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optional: load a pretrained DenseNet model.\n",
    "densenet = torchvision.models.densenet161(pretrained=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select the model (alexnet/densenet)\n",
    "model = densenet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since the model is pretrained, the parameters can be \"frozen\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Freeze the parameters for the pretrained part\n",
    "# Source: https://pytorch.org/docs/master/notes/autograd.html\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Update the last layers of the model, in order to output only 2 classes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DenseNet\n"
     ]
    }
   ],
   "source": [
    "# Note: both models must be loaded, for this to work\n",
    "if model == densenet:\n",
    "    \n",
    "    # Update the classifier layer, in order to output the required number of classes specific to our problem\n",
    "    # Note: see model.eval() for details of each layer\n",
    "    model.classifier = nn.Linear(in_features=2208, out_features=num_classes, bias=True)\n",
    "\n",
    "else:\n",
    "    \n",
    "    # Source: \n",
    "    # https://analyticsindiamag.com/implementing-alexnet-using-pytorch-as-a-transfer-learning-model-in-multi-class-classification/\n",
    "    # Updating the second classifier(reduce the number of outputs, to prevent overfitting)\n",
    "    model.classifier[4] = nn.Linear(4096,1024)\n",
    "\n",
    "    # Updating the third and the last classifier that is the output layer of the network\n",
    "    # Binary classification , thus only 2 output nodes\n",
    "    model.classifier[6] = nn.Linear(1024, num_classes)\n",
    "\n",
    "model_name = model.__class__.__name__\n",
    "print(model_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define loss function: Cross Entropy Loss\n",
    "\n",
    "Note: Improvement option by adding weight class check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Possible Improvement\n",
    "# Source: \n",
    "# https://github.com/choosehappy/PytorchDigitalPathology/blob/master/visualization_densenet/train_densenet.ipynb\n",
    "\n",
    "# \"we have the ability to weight individual classes, in this case we'll do so based on their presense in the trainingset\n",
    "# to avoid biasing any particular class\"\n",
    "# nclasses = dataset[\"train\"].classsizes.shape[0]\n",
    "# class_weight=dataset[\"train\"].classsizes\n",
    "# class_weight = torch.from_numpy(1-class_weight/class_weight.sum()).type('torch.FloatTensor').to(device)\n",
    "# print(class_weight) #show final used weights, make sure that they're reasonable before continouing\n",
    "\n",
    "\n",
    "# criterion = torch.nn.CrossEntropyLoss(weight = class_weight)\n",
    "criterion = nn.CrossEntropyLoss()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define optimizer: Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Source: \n",
    "# https://github.com/choosehappy/PytorchDigitalPathology/blob/master/visualization_densenet/train_densenet.ipynb\n",
    "\n",
    "# adam is going to be the most robust, though perhaps not the best performing, typically a good place to start\n",
    "optimizer = optim.Adam(model.parameters()) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If GPU is available, use it. Otherwise, use the CPU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are running on the following device: cpu\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(f\"You are running on the following device: {device}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Print model and optimizer parameters before training**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Print model's state_dict\n",
    "print(\"Model's state_dict:\")\n",
    "for param_tensor in model.state_dict():\n",
    "    print(param_tensor, \"\\t\", model.state_dict()[param_tensor].size())\n",
    "\n",
    "# Print optimizer's state_dict\n",
    "print(\"Optimizer's state_dict:\")\n",
    "for var_name in optimizer.state_dict():\n",
    "    print(var_name, \"\\t\", optimizer.state_dict()[var_name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "history = []\n",
    "\n",
    "# Initialize the variable where accuracy and loss are stored (with max/min values)\n",
    "history.append([1.0, 1.0, 0.0, 0.0])\n",
    "best_loss_on_val = np.Infinity\n",
    "\n",
    "epochs = 20\n",
    "for epoch in range(epochs):\n",
    "    epoch_start = time.time()\n",
    "    print(\"Epoch: {}/{}\".format(epoch, epochs))\n",
    "    \n",
    "    # Set gradient calculation to ON. Needed during training.\n",
    "    torch.set_grad_enabled(True)\n",
    "        \n",
    "    # Set to training mode\n",
    "    model.train()\n",
    "    \n",
    "    # Loss and Accuracy within the epoch\n",
    "    train_loss = 0.0\n",
    "    train_acc = 0.0\n",
    "     \n",
    "    valid_loss = 0.0\n",
    "    valid_acc = 0.0\n",
    " \n",
    "    # Iterate through all batches of training data\n",
    "    for i, (inputs, labels) in enumerate(train_data):\n",
    " \n",
    "        inputs = inputs.to(device)\n",
    "        labels = labels.to(device)\n",
    "         \n",
    "        # Clean existing gradients\n",
    "        optimizer.zero_grad()\n",
    "         \n",
    "        # Forward pass - compute outputs on input data using the model\n",
    "        outputs = model(inputs)\n",
    "                 \n",
    "        # Compute loss\n",
    "        loss = criterion(outputs, labels)\n",
    "         \n",
    "        # Backpropagate the gradients\n",
    "        loss.backward()\n",
    "         \n",
    "        # Update the parameters\n",
    "        optimizer.step()\n",
    "         \n",
    "        # Compute the total loss for the batch and add it to train_loss\n",
    "        train_loss += loss.item() * inputs.size(0)\n",
    "         \n",
    "        # Compute the accuracy\n",
    "        ret, predictions = torch.max(outputs.data, 1)\n",
    "\n",
    "        correct_counts = predictions.eq(labels.data.view_as(predictions))\n",
    "         \n",
    "        # Convert correct_counts to float and then compute the mean\n",
    "        acc = torch.mean(correct_counts.type(torch.FloatTensor))\n",
    "         \n",
    "        # Compute total accuracy in the whole batch and add to train_acc\n",
    "        train_acc += acc.item() * inputs.size(0)\n",
    "\n",
    "         \n",
    "        print(\"Batch number: {:03d}, Training: Loss: {:.4f}, Accuracy: {:.4f}\".format(i, loss.item(), acc.item()))\n",
    "\n",
    "        # Break \"train_data batch\" for loop\n",
    "        # break\n",
    "    \n",
    "        \n",
    "\n",
    "    # Validation is carried out in each epoch immediately after the training loop\n",
    "    # Validation - No gradient calculation is needed.\n",
    "    with torch.no_grad():\n",
    "\n",
    "        # Set to evaluation mode\n",
    "        model.eval()\n",
    "\n",
    "        # Validation loop\n",
    "        # Iterate through all batches of validation data\n",
    "        for j, (inputs, labels) in enumerate(valid_data):\n",
    "            inputs = inputs.to(device)\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            # Forward pass - compute outputs on input data using the model\n",
    "            outputs = model(inputs)\n",
    "\n",
    "            # Compute loss\n",
    "            loss = criterion(outputs, labels)\n",
    "\n",
    "            # Compute the total loss for the batch and add it to valid_loss\n",
    "            valid_loss += loss.item() * inputs.size(0)\n",
    "\n",
    "            # Calculate validation accuracy\n",
    "            ret, predictions = torch.max(outputs.data, 1)\n",
    "            correct_counts = predictions.eq(labels.data.view_as(predictions))\n",
    "\n",
    "            # Convert correct_counts to float and then compute the mean\n",
    "            acc = torch.mean(correct_counts.type(torch.FloatTensor))\n",
    "\n",
    "            # Compute total accuracy in the whole batch and add to valid_acc\n",
    "            valid_acc += acc.item() * inputs.size(0)\n",
    "\n",
    "\n",
    "            print(\"Validation Batch number: {:03d}, Validation: Loss: {:.4f}, Accuracy: {:.4f}\".format(j, loss.item(), acc.item()))\n",
    "\n",
    "            # Break \"valid_data batch\" for loop\n",
    "            # break\n",
    "\n",
    "    # Find average training loss and training accuracy\n",
    "    avg_train_loss = train_loss/train_data_size\n",
    "    avg_train_acc = train_acc/float(train_data_size)\n",
    "\n",
    "    # Find average validation loss and validation accuracy\n",
    "    avg_valid_loss = valid_loss/valid_data_size\n",
    "    avg_valid_acc = valid_acc/float(valid_data_size)\n",
    "\n",
    "    history.append([avg_train_loss, avg_valid_loss, avg_train_acc, avg_valid_acc])\n",
    "\n",
    "    epoch_end = time.time()\n",
    "\n",
    "    print(f\"Model: {model_name} \\n\")\n",
    "    print(\"Epoch : {:03d} \\nTraining: Loss: {:.4f}, Accuracy: {:.4f}%, \\nValidation : Loss : {:.4f}, Accuracy: {:.4f}%, \\nTime (train+val): {:.4f}s\".format(epoch, avg_train_loss, avg_train_acc*100, avg_valid_loss, avg_valid_acc*100, epoch_end-epoch_start))\n",
    "\n",
    "    \n",
    "    # Source: https://github.com/choosehappy/PytorchDigitalPathology/blob/master/classification_lymphoma_densenet/train_densenet.ipynb\n",
    "    # If current loss is the best we've seen, save model state with all variables\n",
    "    # necessary for recreation\n",
    "    if avg_valid_loss < best_loss_on_val:\n",
    "        best_loss_on_val = avg_valid_loss\n",
    "        print(\"  **\")\n",
    "        state = {'epoch': epoch + 1,\n",
    "         'model_dict': model.state_dict(),\n",
    "         'optim_dict': optimizer.state_dict(),\n",
    "         'loss': criterion,\n",
    "         'best_loss_on_val': best_loss_on_val}\n",
    "\n",
    "        torch.save(state, f\"outputs/{model_name}_best_model.pth\")\n",
    "        print(f\"Saved model {model_name} with loss {avg_valid_loss}\")\n",
    "    else:\n",
    "        print(\"\")\n",
    "    \n",
    "    # Stop \"epoch\" for\n",
    "    # break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Take a look at the model structure**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DenseNet(\n",
      "  (features): Sequential(\n",
      "    (conv0): Conv2d(3, 96, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
      "    (norm0): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (relu0): ReLU(inplace=True)\n",
      "    (pool0): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
      "    (denseblock1): _DenseBlock(\n",
      "      (denselayer1): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(96, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer2): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(144, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer3): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer4): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(240, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer5): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(288, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer6): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(336, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(336, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "    )\n",
      "    (transition1): _Transition(\n",
      "      (norm): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace=True)\n",
      "      (conv): Conv2d(384, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
      "    )\n",
      "    (denseblock2): _DenseBlock(\n",
      "      (denselayer1): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer2): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(240, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer3): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(288, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer4): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(336, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(336, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer5): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(384, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer6): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(432, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(432, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer7): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(480, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer8): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(528, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(528, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer9): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(576, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer10): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(624, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(624, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer11): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(672, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer12): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(720, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(720, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "    )\n",
      "    (transition2): _Transition(\n",
      "      (norm): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace=True)\n",
      "      (conv): Conv2d(768, 384, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
      "    )\n",
      "    (denseblock3): _DenseBlock(\n",
      "      (denselayer1): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(384, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer2): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(432, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(432, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer3): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(480, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer4): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(528, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(528, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer5): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(576, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer6): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(624, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(624, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer7): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(672, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer8): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(720, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(720, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer9): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer10): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(816, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer11): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(864, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(864, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer12): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(912, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(912, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer13): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(960, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(960, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer14): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1008, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1008, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer15): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1056, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1056, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer16): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1104, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1104, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer17): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer18): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1200, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer19): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1248, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1248, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer20): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1296, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1296, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer21): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1344, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1344, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer22): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1392, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1392, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer23): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1440, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1440, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer24): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1488, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1488, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer25): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1536, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer26): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1584, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1584, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer27): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1632, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1632, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer28): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1680, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1680, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer29): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1728, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1728, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer30): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1776, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1776, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer31): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1824, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1824, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer32): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1872, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1872, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer33): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1920, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1920, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer34): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1968, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1968, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer35): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(2016, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(2016, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer36): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(2064, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(2064, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "    )\n",
      "    (transition3): _Transition(\n",
      "      (norm): BatchNorm2d(2112, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace=True)\n",
      "      (conv): Conv2d(2112, 1056, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
      "    )\n",
      "    (denseblock4): _DenseBlock(\n",
      "      (denselayer1): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1056, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1056, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer2): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1104, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1104, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer3): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer4): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1200, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer5): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1248, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1248, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer6): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1296, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1296, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer7): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1344, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1344, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer8): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1392, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1392, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer9): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1440, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1440, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer10): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1488, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1488, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer11): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1536, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer12): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1584, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1584, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer13): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1632, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1632, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer14): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1680, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1680, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer15): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1728, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1728, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer16): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1776, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1776, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer17): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1824, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1824, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer18): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1872, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1872, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer19): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1920, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1920, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer20): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(1968, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(1968, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer21): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(2016, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(2016, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer22): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(2064, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(2064, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer23): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(2112, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(2112, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer24): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(2160, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(2160, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(192, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "    )\n",
      "    (norm5): BatchNorm2d(2208, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  )\n",
      "  (classifier): Linear(in_features=2208, out_features=2, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Plot the accuracy and loss**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "\n",
    "matplotlib.style.use('ggplot')\n",
    "\n",
    "# accuracy plots\n",
    "plt.figure(figsize=(10, 7))\n",
    "plt.plot(range(len(history)), [x[3] for x in history] , color='green', label='validation accuracy')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()\n",
    "plt.savefig('outputs/initial_validation_accuracy.png')\n",
    "plt.show()\n",
    " \n",
    "# loss plots\n",
    "plt.figure(figsize=(10, 7))\n",
    "plt.plot(range(len(history)), [x[1] for x in history], color='orange', label='validation loss')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()\n",
    "plt.savefig('outputs/initial_training_loss.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Print model and optimizer parameters after training**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print model's state_dict\n",
    "print(\"Model's state_dict:\")\n",
    "for param_tensor in model.state_dict():\n",
    "    print(param_tensor, \"\\t\", model.state_dict()[param_tensor].size())\n",
    "\n",
    "# Print optimizer's state_dict\n",
    "print(\"Optimizer's state_dict:\")\n",
    "for var_name in optimizer.state_dict():\n",
    "    print(var_name, \"\\t\", optimizer.state_dict()[var_name])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Save the model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Source:\n",
    "# https://debuggercafe.com/effective-model-saving-and-resuming-training-in-pytorch/\n",
    "# save model checkpoint\n",
    "torch.save({\n",
    "            'epoch': epochs,\n",
    "            'model_dict': model.state_dict(),\n",
    "            'optim_dict': optimizer.state_dict(),\n",
    "            'loss': criterion,\n",
    "            }, f'outputs/{model_name}_trained.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Initialize the model before loading the previously saved one**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "if model == alexnet:\n",
    "\n",
    "    # Initialize the model\n",
    "    model_loaded = torchvision.models.alexnet(pretrained=True)\n",
    "    \n",
    "    # Updating the second classifier\n",
    "    model_loaded.classifier[4] = nn.Linear(4096,1024)\n",
    "\n",
    "    # Binary classification , thus only 2 output nodes\n",
    "    model_loaded.classifier[6] = nn.Linear(1024, num_classes)\n",
    "    \n",
    "if model == densenet:\n",
    "    \n",
    "    # Optional: load a pretrained DenseNet model.\n",
    "    model_loaded = torchvision.models.densenet161(pretrained=True)\n",
    "    \n",
    "    # Update the classifier layer, in order to output the required number of classes specific to our problem\n",
    "    # Note: see model.eval() for details of each layer\n",
    "    model_loaded.classifier = nn.Linear(in_features=2208, out_features=num_classes, bias=True)\n",
    "    \n",
    "    \n",
    "# Initialize optimizer  before loading optimizer state_dict\n",
    "optimizer_loaded = optim.Adam(model_loaded.parameters()) \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load the saved model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Previously trained model weights state_dict loaded...\n",
      "Previously trained for 2 number of epochs...\n"
     ]
    }
   ],
   "source": [
    "# load the model checkpoint\n",
    "# checkpoint = torch.load(f'outputs/{model_name}_trained.pth')\n",
    "checkpoint = torch.load(f'outputs/{model_name}_best_model.pth')\n",
    "\n",
    "# load model weights state_dict\n",
    "model_loaded.load_state_dict(checkpoint['model_dict'])\n",
    "print('Previously trained model weights state_dict loaded...')\n",
    "\n",
    "# load trained optimizer state_dict\n",
    "# optimizer_loaded.load_state_dict(checkpoint['optim_dict'])\n",
    "# print('Previously trained optimizer state_dict loaded...')\n",
    "\n",
    "epochs = checkpoint['epoch']\n",
    "\n",
    "# load the criterion\n",
    "# criterion_loaded = checkpoint['loss']\n",
    "# print('Trained model loss function loaded...')\n",
    "\n",
    "print(f\"Previously trained for {epochs} number of epochs...\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Print loaded model and optimizer parameters**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model's state_dict:\n",
      "features.conv0.weight \t torch.Size([96, 3, 7, 7])\n",
      "features.norm0.weight \t torch.Size([96])\n",
      "features.norm0.bias \t torch.Size([96])\n",
      "features.norm0.running_mean \t torch.Size([96])\n",
      "features.norm0.running_var \t torch.Size([96])\n",
      "features.norm0.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock1.denselayer1.norm1.weight \t torch.Size([96])\n",
      "features.denseblock1.denselayer1.norm1.bias \t torch.Size([96])\n",
      "features.denseblock1.denselayer1.norm1.running_mean \t torch.Size([96])\n",
      "features.denseblock1.denselayer1.norm1.running_var \t torch.Size([96])\n",
      "features.denseblock1.denselayer1.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock1.denselayer1.conv1.weight \t torch.Size([192, 96, 1, 1])\n",
      "features.denseblock1.denselayer1.norm2.weight \t torch.Size([192])\n",
      "features.denseblock1.denselayer1.norm2.bias \t torch.Size([192])\n",
      "features.denseblock1.denselayer1.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock1.denselayer1.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock1.denselayer1.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock1.denselayer1.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock1.denselayer2.norm1.weight \t torch.Size([144])\n",
      "features.denseblock1.denselayer2.norm1.bias \t torch.Size([144])\n",
      "features.denseblock1.denselayer2.norm1.running_mean \t torch.Size([144])\n",
      "features.denseblock1.denselayer2.norm1.running_var \t torch.Size([144])\n",
      "features.denseblock1.denselayer2.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock1.denselayer2.conv1.weight \t torch.Size([192, 144, 1, 1])\n",
      "features.denseblock1.denselayer2.norm2.weight \t torch.Size([192])\n",
      "features.denseblock1.denselayer2.norm2.bias \t torch.Size([192])\n",
      "features.denseblock1.denselayer2.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock1.denselayer2.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock1.denselayer2.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock1.denselayer2.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock1.denselayer3.norm1.weight \t torch.Size([192])\n",
      "features.denseblock1.denselayer3.norm1.bias \t torch.Size([192])\n",
      "features.denseblock1.denselayer3.norm1.running_mean \t torch.Size([192])\n",
      "features.denseblock1.denselayer3.norm1.running_var \t torch.Size([192])\n",
      "features.denseblock1.denselayer3.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock1.denselayer3.conv1.weight \t torch.Size([192, 192, 1, 1])\n",
      "features.denseblock1.denselayer3.norm2.weight \t torch.Size([192])\n",
      "features.denseblock1.denselayer3.norm2.bias \t torch.Size([192])\n",
      "features.denseblock1.denselayer3.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock1.denselayer3.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock1.denselayer3.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock1.denselayer3.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock1.denselayer4.norm1.weight \t torch.Size([240])\n",
      "features.denseblock1.denselayer4.norm1.bias \t torch.Size([240])\n",
      "features.denseblock1.denselayer4.norm1.running_mean \t torch.Size([240])\n",
      "features.denseblock1.denselayer4.norm1.running_var \t torch.Size([240])\n",
      "features.denseblock1.denselayer4.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock1.denselayer4.conv1.weight \t torch.Size([192, 240, 1, 1])\n",
      "features.denseblock1.denselayer4.norm2.weight \t torch.Size([192])\n",
      "features.denseblock1.denselayer4.norm2.bias \t torch.Size([192])\n",
      "features.denseblock1.denselayer4.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock1.denselayer4.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock1.denselayer4.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock1.denselayer4.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock1.denselayer5.norm1.weight \t torch.Size([288])\n",
      "features.denseblock1.denselayer5.norm1.bias \t torch.Size([288])\n",
      "features.denseblock1.denselayer5.norm1.running_mean \t torch.Size([288])\n",
      "features.denseblock1.denselayer5.norm1.running_var \t torch.Size([288])\n",
      "features.denseblock1.denselayer5.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock1.denselayer5.conv1.weight \t torch.Size([192, 288, 1, 1])\n",
      "features.denseblock1.denselayer5.norm2.weight \t torch.Size([192])\n",
      "features.denseblock1.denselayer5.norm2.bias \t torch.Size([192])\n",
      "features.denseblock1.denselayer5.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock1.denselayer5.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock1.denselayer5.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock1.denselayer5.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock1.denselayer6.norm1.weight \t torch.Size([336])\n",
      "features.denseblock1.denselayer6.norm1.bias \t torch.Size([336])\n",
      "features.denseblock1.denselayer6.norm1.running_mean \t torch.Size([336])\n",
      "features.denseblock1.denselayer6.norm1.running_var \t torch.Size([336])\n",
      "features.denseblock1.denselayer6.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock1.denselayer6.conv1.weight \t torch.Size([192, 336, 1, 1])\n",
      "features.denseblock1.denselayer6.norm2.weight \t torch.Size([192])\n",
      "features.denseblock1.denselayer6.norm2.bias \t torch.Size([192])\n",
      "features.denseblock1.denselayer6.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock1.denselayer6.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock1.denselayer6.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock1.denselayer6.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.transition1.norm.weight \t torch.Size([384])\n",
      "features.transition1.norm.bias \t torch.Size([384])\n",
      "features.transition1.norm.running_mean \t torch.Size([384])\n",
      "features.transition1.norm.running_var \t torch.Size([384])\n",
      "features.transition1.norm.num_batches_tracked \t torch.Size([])\n",
      "features.transition1.conv.weight \t torch.Size([192, 384, 1, 1])\n",
      "features.denseblock2.denselayer1.norm1.weight \t torch.Size([192])\n",
      "features.denseblock2.denselayer1.norm1.bias \t torch.Size([192])\n",
      "features.denseblock2.denselayer1.norm1.running_mean \t torch.Size([192])\n",
      "features.denseblock2.denselayer1.norm1.running_var \t torch.Size([192])\n",
      "features.denseblock2.denselayer1.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer1.conv1.weight \t torch.Size([192, 192, 1, 1])\n",
      "features.denseblock2.denselayer1.norm2.weight \t torch.Size([192])\n",
      "features.denseblock2.denselayer1.norm2.bias \t torch.Size([192])\n",
      "features.denseblock2.denselayer1.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock2.denselayer1.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock2.denselayer1.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer1.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock2.denselayer2.norm1.weight \t torch.Size([240])\n",
      "features.denseblock2.denselayer2.norm1.bias \t torch.Size([240])\n",
      "features.denseblock2.denselayer2.norm1.running_mean \t torch.Size([240])\n",
      "features.denseblock2.denselayer2.norm1.running_var \t torch.Size([240])\n",
      "features.denseblock2.denselayer2.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer2.conv1.weight \t torch.Size([192, 240, 1, 1])\n",
      "features.denseblock2.denselayer2.norm2.weight \t torch.Size([192])\n",
      "features.denseblock2.denselayer2.norm2.bias \t torch.Size([192])\n",
      "features.denseblock2.denselayer2.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock2.denselayer2.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock2.denselayer2.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer2.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock2.denselayer3.norm1.weight \t torch.Size([288])\n",
      "features.denseblock2.denselayer3.norm1.bias \t torch.Size([288])\n",
      "features.denseblock2.denselayer3.norm1.running_mean \t torch.Size([288])\n",
      "features.denseblock2.denselayer3.norm1.running_var \t torch.Size([288])\n",
      "features.denseblock2.denselayer3.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer3.conv1.weight \t torch.Size([192, 288, 1, 1])\n",
      "features.denseblock2.denselayer3.norm2.weight \t torch.Size([192])\n",
      "features.denseblock2.denselayer3.norm2.bias \t torch.Size([192])\n",
      "features.denseblock2.denselayer3.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock2.denselayer3.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock2.denselayer3.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer3.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock2.denselayer4.norm1.weight \t torch.Size([336])\n",
      "features.denseblock2.denselayer4.norm1.bias \t torch.Size([336])\n",
      "features.denseblock2.denselayer4.norm1.running_mean \t torch.Size([336])\n",
      "features.denseblock2.denselayer4.norm1.running_var \t torch.Size([336])\n",
      "features.denseblock2.denselayer4.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer4.conv1.weight \t torch.Size([192, 336, 1, 1])\n",
      "features.denseblock2.denselayer4.norm2.weight \t torch.Size([192])\n",
      "features.denseblock2.denselayer4.norm2.bias \t torch.Size([192])\n",
      "features.denseblock2.denselayer4.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock2.denselayer4.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock2.denselayer4.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer4.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock2.denselayer5.norm1.weight \t torch.Size([384])\n",
      "features.denseblock2.denselayer5.norm1.bias \t torch.Size([384])\n",
      "features.denseblock2.denselayer5.norm1.running_mean \t torch.Size([384])\n",
      "features.denseblock2.denselayer5.norm1.running_var \t torch.Size([384])\n",
      "features.denseblock2.denselayer5.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer5.conv1.weight \t torch.Size([192, 384, 1, 1])\n",
      "features.denseblock2.denselayer5.norm2.weight \t torch.Size([192])\n",
      "features.denseblock2.denselayer5.norm2.bias \t torch.Size([192])\n",
      "features.denseblock2.denselayer5.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock2.denselayer5.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock2.denselayer5.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer5.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock2.denselayer6.norm1.weight \t torch.Size([432])\n",
      "features.denseblock2.denselayer6.norm1.bias \t torch.Size([432])\n",
      "features.denseblock2.denselayer6.norm1.running_mean \t torch.Size([432])\n",
      "features.denseblock2.denselayer6.norm1.running_var \t torch.Size([432])\n",
      "features.denseblock2.denselayer6.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer6.conv1.weight \t torch.Size([192, 432, 1, 1])\n",
      "features.denseblock2.denselayer6.norm2.weight \t torch.Size([192])\n",
      "features.denseblock2.denselayer6.norm2.bias \t torch.Size([192])\n",
      "features.denseblock2.denselayer6.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock2.denselayer6.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock2.denselayer6.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer6.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock2.denselayer7.norm1.weight \t torch.Size([480])\n",
      "features.denseblock2.denselayer7.norm1.bias \t torch.Size([480])\n",
      "features.denseblock2.denselayer7.norm1.running_mean \t torch.Size([480])\n",
      "features.denseblock2.denselayer7.norm1.running_var \t torch.Size([480])\n",
      "features.denseblock2.denselayer7.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer7.conv1.weight \t torch.Size([192, 480, 1, 1])\n",
      "features.denseblock2.denselayer7.norm2.weight \t torch.Size([192])\n",
      "features.denseblock2.denselayer7.norm2.bias \t torch.Size([192])\n",
      "features.denseblock2.denselayer7.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock2.denselayer7.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock2.denselayer7.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer7.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock2.denselayer8.norm1.weight \t torch.Size([528])\n",
      "features.denseblock2.denselayer8.norm1.bias \t torch.Size([528])\n",
      "features.denseblock2.denselayer8.norm1.running_mean \t torch.Size([528])\n",
      "features.denseblock2.denselayer8.norm1.running_var \t torch.Size([528])\n",
      "features.denseblock2.denselayer8.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer8.conv1.weight \t torch.Size([192, 528, 1, 1])\n",
      "features.denseblock2.denselayer8.norm2.weight \t torch.Size([192])\n",
      "features.denseblock2.denselayer8.norm2.bias \t torch.Size([192])\n",
      "features.denseblock2.denselayer8.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock2.denselayer8.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock2.denselayer8.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer8.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock2.denselayer9.norm1.weight \t torch.Size([576])\n",
      "features.denseblock2.denselayer9.norm1.bias \t torch.Size([576])\n",
      "features.denseblock2.denselayer9.norm1.running_mean \t torch.Size([576])\n",
      "features.denseblock2.denselayer9.norm1.running_var \t torch.Size([576])\n",
      "features.denseblock2.denselayer9.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer9.conv1.weight \t torch.Size([192, 576, 1, 1])\n",
      "features.denseblock2.denselayer9.norm2.weight \t torch.Size([192])\n",
      "features.denseblock2.denselayer9.norm2.bias \t torch.Size([192])\n",
      "features.denseblock2.denselayer9.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock2.denselayer9.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock2.denselayer9.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer9.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock2.denselayer10.norm1.weight \t torch.Size([624])\n",
      "features.denseblock2.denselayer10.norm1.bias \t torch.Size([624])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features.denseblock2.denselayer10.norm1.running_mean \t torch.Size([624])\n",
      "features.denseblock2.denselayer10.norm1.running_var \t torch.Size([624])\n",
      "features.denseblock2.denselayer10.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer10.conv1.weight \t torch.Size([192, 624, 1, 1])\n",
      "features.denseblock2.denselayer10.norm2.weight \t torch.Size([192])\n",
      "features.denseblock2.denselayer10.norm2.bias \t torch.Size([192])\n",
      "features.denseblock2.denselayer10.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock2.denselayer10.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock2.denselayer10.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer10.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock2.denselayer11.norm1.weight \t torch.Size([672])\n",
      "features.denseblock2.denselayer11.norm1.bias \t torch.Size([672])\n",
      "features.denseblock2.denselayer11.norm1.running_mean \t torch.Size([672])\n",
      "features.denseblock2.denselayer11.norm1.running_var \t torch.Size([672])\n",
      "features.denseblock2.denselayer11.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer11.conv1.weight \t torch.Size([192, 672, 1, 1])\n",
      "features.denseblock2.denselayer11.norm2.weight \t torch.Size([192])\n",
      "features.denseblock2.denselayer11.norm2.bias \t torch.Size([192])\n",
      "features.denseblock2.denselayer11.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock2.denselayer11.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock2.denselayer11.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer11.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock2.denselayer12.norm1.weight \t torch.Size([720])\n",
      "features.denseblock2.denselayer12.norm1.bias \t torch.Size([720])\n",
      "features.denseblock2.denselayer12.norm1.running_mean \t torch.Size([720])\n",
      "features.denseblock2.denselayer12.norm1.running_var \t torch.Size([720])\n",
      "features.denseblock2.denselayer12.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer12.conv1.weight \t torch.Size([192, 720, 1, 1])\n",
      "features.denseblock2.denselayer12.norm2.weight \t torch.Size([192])\n",
      "features.denseblock2.denselayer12.norm2.bias \t torch.Size([192])\n",
      "features.denseblock2.denselayer12.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock2.denselayer12.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock2.denselayer12.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock2.denselayer12.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.transition2.norm.weight \t torch.Size([768])\n",
      "features.transition2.norm.bias \t torch.Size([768])\n",
      "features.transition2.norm.running_mean \t torch.Size([768])\n",
      "features.transition2.norm.running_var \t torch.Size([768])\n",
      "features.transition2.norm.num_batches_tracked \t torch.Size([])\n",
      "features.transition2.conv.weight \t torch.Size([384, 768, 1, 1])\n",
      "features.denseblock3.denselayer1.norm1.weight \t torch.Size([384])\n",
      "features.denseblock3.denselayer1.norm1.bias \t torch.Size([384])\n",
      "features.denseblock3.denselayer1.norm1.running_mean \t torch.Size([384])\n",
      "features.denseblock3.denselayer1.norm1.running_var \t torch.Size([384])\n",
      "features.denseblock3.denselayer1.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer1.conv1.weight \t torch.Size([192, 384, 1, 1])\n",
      "features.denseblock3.denselayer1.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer1.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer1.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer1.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer1.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer1.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer2.norm1.weight \t torch.Size([432])\n",
      "features.denseblock3.denselayer2.norm1.bias \t torch.Size([432])\n",
      "features.denseblock3.denselayer2.norm1.running_mean \t torch.Size([432])\n",
      "features.denseblock3.denselayer2.norm1.running_var \t torch.Size([432])\n",
      "features.denseblock3.denselayer2.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer2.conv1.weight \t torch.Size([192, 432, 1, 1])\n",
      "features.denseblock3.denselayer2.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer2.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer2.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer2.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer2.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer2.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer3.norm1.weight \t torch.Size([480])\n",
      "features.denseblock3.denselayer3.norm1.bias \t torch.Size([480])\n",
      "features.denseblock3.denselayer3.norm1.running_mean \t torch.Size([480])\n",
      "features.denseblock3.denselayer3.norm1.running_var \t torch.Size([480])\n",
      "features.denseblock3.denselayer3.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer3.conv1.weight \t torch.Size([192, 480, 1, 1])\n",
      "features.denseblock3.denselayer3.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer3.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer3.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer3.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer3.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer3.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer4.norm1.weight \t torch.Size([528])\n",
      "features.denseblock3.denselayer4.norm1.bias \t torch.Size([528])\n",
      "features.denseblock3.denselayer4.norm1.running_mean \t torch.Size([528])\n",
      "features.denseblock3.denselayer4.norm1.running_var \t torch.Size([528])\n",
      "features.denseblock3.denselayer4.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer4.conv1.weight \t torch.Size([192, 528, 1, 1])\n",
      "features.denseblock3.denselayer4.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer4.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer4.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer4.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer4.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer4.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer5.norm1.weight \t torch.Size([576])\n",
      "features.denseblock3.denselayer5.norm1.bias \t torch.Size([576])\n",
      "features.denseblock3.denselayer5.norm1.running_mean \t torch.Size([576])\n",
      "features.denseblock3.denselayer5.norm1.running_var \t torch.Size([576])\n",
      "features.denseblock3.denselayer5.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer5.conv1.weight \t torch.Size([192, 576, 1, 1])\n",
      "features.denseblock3.denselayer5.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer5.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer5.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer5.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer5.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer5.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer6.norm1.weight \t torch.Size([624])\n",
      "features.denseblock3.denselayer6.norm1.bias \t torch.Size([624])\n",
      "features.denseblock3.denselayer6.norm1.running_mean \t torch.Size([624])\n",
      "features.denseblock3.denselayer6.norm1.running_var \t torch.Size([624])\n",
      "features.denseblock3.denselayer6.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer6.conv1.weight \t torch.Size([192, 624, 1, 1])\n",
      "features.denseblock3.denselayer6.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer6.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer6.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer6.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer6.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer6.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer7.norm1.weight \t torch.Size([672])\n",
      "features.denseblock3.denselayer7.norm1.bias \t torch.Size([672])\n",
      "features.denseblock3.denselayer7.norm1.running_mean \t torch.Size([672])\n",
      "features.denseblock3.denselayer7.norm1.running_var \t torch.Size([672])\n",
      "features.denseblock3.denselayer7.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer7.conv1.weight \t torch.Size([192, 672, 1, 1])\n",
      "features.denseblock3.denselayer7.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer7.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer7.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer7.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer7.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer7.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer8.norm1.weight \t torch.Size([720])\n",
      "features.denseblock3.denselayer8.norm1.bias \t torch.Size([720])\n",
      "features.denseblock3.denselayer8.norm1.running_mean \t torch.Size([720])\n",
      "features.denseblock3.denselayer8.norm1.running_var \t torch.Size([720])\n",
      "features.denseblock3.denselayer8.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer8.conv1.weight \t torch.Size([192, 720, 1, 1])\n",
      "features.denseblock3.denselayer8.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer8.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer8.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer8.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer8.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer8.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer9.norm1.weight \t torch.Size([768])\n",
      "features.denseblock3.denselayer9.norm1.bias \t torch.Size([768])\n",
      "features.denseblock3.denselayer9.norm1.running_mean \t torch.Size([768])\n",
      "features.denseblock3.denselayer9.norm1.running_var \t torch.Size([768])\n",
      "features.denseblock3.denselayer9.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer9.conv1.weight \t torch.Size([192, 768, 1, 1])\n",
      "features.denseblock3.denselayer9.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer9.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer9.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer9.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer9.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer9.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer10.norm1.weight \t torch.Size([816])\n",
      "features.denseblock3.denselayer10.norm1.bias \t torch.Size([816])\n",
      "features.denseblock3.denselayer10.norm1.running_mean \t torch.Size([816])\n",
      "features.denseblock3.denselayer10.norm1.running_var \t torch.Size([816])\n",
      "features.denseblock3.denselayer10.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer10.conv1.weight \t torch.Size([192, 816, 1, 1])\n",
      "features.denseblock3.denselayer10.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer10.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer10.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer10.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer10.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer10.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer11.norm1.weight \t torch.Size([864])\n",
      "features.denseblock3.denselayer11.norm1.bias \t torch.Size([864])\n",
      "features.denseblock3.denselayer11.norm1.running_mean \t torch.Size([864])\n",
      "features.denseblock3.denselayer11.norm1.running_var \t torch.Size([864])\n",
      "features.denseblock3.denselayer11.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer11.conv1.weight \t torch.Size([192, 864, 1, 1])\n",
      "features.denseblock3.denselayer11.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer11.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer11.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer11.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer11.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer11.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer12.norm1.weight \t torch.Size([912])\n",
      "features.denseblock3.denselayer12.norm1.bias \t torch.Size([912])\n",
      "features.denseblock3.denselayer12.norm1.running_mean \t torch.Size([912])\n",
      "features.denseblock3.denselayer12.norm1.running_var \t torch.Size([912])\n",
      "features.denseblock3.denselayer12.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer12.conv1.weight \t torch.Size([192, 912, 1, 1])\n",
      "features.denseblock3.denselayer12.norm2.weight \t torch.Size([192])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features.denseblock3.denselayer12.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer12.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer12.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer12.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer12.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer13.norm1.weight \t torch.Size([960])\n",
      "features.denseblock3.denselayer13.norm1.bias \t torch.Size([960])\n",
      "features.denseblock3.denselayer13.norm1.running_mean \t torch.Size([960])\n",
      "features.denseblock3.denselayer13.norm1.running_var \t torch.Size([960])\n",
      "features.denseblock3.denselayer13.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer13.conv1.weight \t torch.Size([192, 960, 1, 1])\n",
      "features.denseblock3.denselayer13.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer13.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer13.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer13.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer13.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer13.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer14.norm1.weight \t torch.Size([1008])\n",
      "features.denseblock3.denselayer14.norm1.bias \t torch.Size([1008])\n",
      "features.denseblock3.denselayer14.norm1.running_mean \t torch.Size([1008])\n",
      "features.denseblock3.denselayer14.norm1.running_var \t torch.Size([1008])\n",
      "features.denseblock3.denselayer14.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer14.conv1.weight \t torch.Size([192, 1008, 1, 1])\n",
      "features.denseblock3.denselayer14.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer14.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer14.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer14.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer14.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer14.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer15.norm1.weight \t torch.Size([1056])\n",
      "features.denseblock3.denselayer15.norm1.bias \t torch.Size([1056])\n",
      "features.denseblock3.denselayer15.norm1.running_mean \t torch.Size([1056])\n",
      "features.denseblock3.denselayer15.norm1.running_var \t torch.Size([1056])\n",
      "features.denseblock3.denselayer15.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer15.conv1.weight \t torch.Size([192, 1056, 1, 1])\n",
      "features.denseblock3.denselayer15.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer15.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer15.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer15.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer15.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer15.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer16.norm1.weight \t torch.Size([1104])\n",
      "features.denseblock3.denselayer16.norm1.bias \t torch.Size([1104])\n",
      "features.denseblock3.denselayer16.norm1.running_mean \t torch.Size([1104])\n",
      "features.denseblock3.denselayer16.norm1.running_var \t torch.Size([1104])\n",
      "features.denseblock3.denselayer16.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer16.conv1.weight \t torch.Size([192, 1104, 1, 1])\n",
      "features.denseblock3.denselayer16.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer16.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer16.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer16.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer16.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer16.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer17.norm1.weight \t torch.Size([1152])\n",
      "features.denseblock3.denselayer17.norm1.bias \t torch.Size([1152])\n",
      "features.denseblock3.denselayer17.norm1.running_mean \t torch.Size([1152])\n",
      "features.denseblock3.denselayer17.norm1.running_var \t torch.Size([1152])\n",
      "features.denseblock3.denselayer17.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer17.conv1.weight \t torch.Size([192, 1152, 1, 1])\n",
      "features.denseblock3.denselayer17.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer17.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer17.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer17.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer17.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer17.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer18.norm1.weight \t torch.Size([1200])\n",
      "features.denseblock3.denselayer18.norm1.bias \t torch.Size([1200])\n",
      "features.denseblock3.denselayer18.norm1.running_mean \t torch.Size([1200])\n",
      "features.denseblock3.denselayer18.norm1.running_var \t torch.Size([1200])\n",
      "features.denseblock3.denselayer18.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer18.conv1.weight \t torch.Size([192, 1200, 1, 1])\n",
      "features.denseblock3.denselayer18.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer18.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer18.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer18.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer18.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer18.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer19.norm1.weight \t torch.Size([1248])\n",
      "features.denseblock3.denselayer19.norm1.bias \t torch.Size([1248])\n",
      "features.denseblock3.denselayer19.norm1.running_mean \t torch.Size([1248])\n",
      "features.denseblock3.denselayer19.norm1.running_var \t torch.Size([1248])\n",
      "features.denseblock3.denselayer19.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer19.conv1.weight \t torch.Size([192, 1248, 1, 1])\n",
      "features.denseblock3.denselayer19.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer19.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer19.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer19.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer19.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer19.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer20.norm1.weight \t torch.Size([1296])\n",
      "features.denseblock3.denselayer20.norm1.bias \t torch.Size([1296])\n",
      "features.denseblock3.denselayer20.norm1.running_mean \t torch.Size([1296])\n",
      "features.denseblock3.denselayer20.norm1.running_var \t torch.Size([1296])\n",
      "features.denseblock3.denselayer20.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer20.conv1.weight \t torch.Size([192, 1296, 1, 1])\n",
      "features.denseblock3.denselayer20.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer20.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer20.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer20.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer20.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer20.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer21.norm1.weight \t torch.Size([1344])\n",
      "features.denseblock3.denselayer21.norm1.bias \t torch.Size([1344])\n",
      "features.denseblock3.denselayer21.norm1.running_mean \t torch.Size([1344])\n",
      "features.denseblock3.denselayer21.norm1.running_var \t torch.Size([1344])\n",
      "features.denseblock3.denselayer21.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer21.conv1.weight \t torch.Size([192, 1344, 1, 1])\n",
      "features.denseblock3.denselayer21.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer21.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer21.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer21.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer21.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer21.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer22.norm1.weight \t torch.Size([1392])\n",
      "features.denseblock3.denselayer22.norm1.bias \t torch.Size([1392])\n",
      "features.denseblock3.denselayer22.norm1.running_mean \t torch.Size([1392])\n",
      "features.denseblock3.denselayer22.norm1.running_var \t torch.Size([1392])\n",
      "features.denseblock3.denselayer22.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer22.conv1.weight \t torch.Size([192, 1392, 1, 1])\n",
      "features.denseblock3.denselayer22.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer22.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer22.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer22.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer22.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer22.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer23.norm1.weight \t torch.Size([1440])\n",
      "features.denseblock3.denselayer23.norm1.bias \t torch.Size([1440])\n",
      "features.denseblock3.denselayer23.norm1.running_mean \t torch.Size([1440])\n",
      "features.denseblock3.denselayer23.norm1.running_var \t torch.Size([1440])\n",
      "features.denseblock3.denselayer23.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer23.conv1.weight \t torch.Size([192, 1440, 1, 1])\n",
      "features.denseblock3.denselayer23.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer23.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer23.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer23.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer23.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer23.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer24.norm1.weight \t torch.Size([1488])\n",
      "features.denseblock3.denselayer24.norm1.bias \t torch.Size([1488])\n",
      "features.denseblock3.denselayer24.norm1.running_mean \t torch.Size([1488])\n",
      "features.denseblock3.denselayer24.norm1.running_var \t torch.Size([1488])\n",
      "features.denseblock3.denselayer24.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer24.conv1.weight \t torch.Size([192, 1488, 1, 1])\n",
      "features.denseblock3.denselayer24.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer24.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer24.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer24.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer24.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer24.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer25.norm1.weight \t torch.Size([1536])\n",
      "features.denseblock3.denselayer25.norm1.bias \t torch.Size([1536])\n",
      "features.denseblock3.denselayer25.norm1.running_mean \t torch.Size([1536])\n",
      "features.denseblock3.denselayer25.norm1.running_var \t torch.Size([1536])\n",
      "features.denseblock3.denselayer25.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer25.conv1.weight \t torch.Size([192, 1536, 1, 1])\n",
      "features.denseblock3.denselayer25.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer25.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer25.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer25.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer25.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer25.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer26.norm1.weight \t torch.Size([1584])\n",
      "features.denseblock3.denselayer26.norm1.bias \t torch.Size([1584])\n",
      "features.denseblock3.denselayer26.norm1.running_mean \t torch.Size([1584])\n",
      "features.denseblock3.denselayer26.norm1.running_var \t torch.Size([1584])\n",
      "features.denseblock3.denselayer26.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer26.conv1.weight \t torch.Size([192, 1584, 1, 1])\n",
      "features.denseblock3.denselayer26.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer26.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer26.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer26.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer26.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer26.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer27.norm1.weight \t torch.Size([1632])\n",
      "features.denseblock3.denselayer27.norm1.bias \t torch.Size([1632])\n",
      "features.denseblock3.denselayer27.norm1.running_mean \t torch.Size([1632])\n",
      "features.denseblock3.denselayer27.norm1.running_var \t torch.Size([1632])\n",
      "features.denseblock3.denselayer27.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer27.conv1.weight \t torch.Size([192, 1632, 1, 1])\n",
      "features.denseblock3.denselayer27.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer27.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer27.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer27.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer27.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer27.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer28.norm1.weight \t torch.Size([1680])\n",
      "features.denseblock3.denselayer28.norm1.bias \t torch.Size([1680])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features.denseblock3.denselayer28.norm1.running_mean \t torch.Size([1680])\n",
      "features.denseblock3.denselayer28.norm1.running_var \t torch.Size([1680])\n",
      "features.denseblock3.denselayer28.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer28.conv1.weight \t torch.Size([192, 1680, 1, 1])\n",
      "features.denseblock3.denselayer28.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer28.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer28.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer28.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer28.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer28.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer29.norm1.weight \t torch.Size([1728])\n",
      "features.denseblock3.denselayer29.norm1.bias \t torch.Size([1728])\n",
      "features.denseblock3.denselayer29.norm1.running_mean \t torch.Size([1728])\n",
      "features.denseblock3.denselayer29.norm1.running_var \t torch.Size([1728])\n",
      "features.denseblock3.denselayer29.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer29.conv1.weight \t torch.Size([192, 1728, 1, 1])\n",
      "features.denseblock3.denselayer29.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer29.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer29.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer29.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer29.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer29.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer30.norm1.weight \t torch.Size([1776])\n",
      "features.denseblock3.denselayer30.norm1.bias \t torch.Size([1776])\n",
      "features.denseblock3.denselayer30.norm1.running_mean \t torch.Size([1776])\n",
      "features.denseblock3.denselayer30.norm1.running_var \t torch.Size([1776])\n",
      "features.denseblock3.denselayer30.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer30.conv1.weight \t torch.Size([192, 1776, 1, 1])\n",
      "features.denseblock3.denselayer30.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer30.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer30.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer30.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer30.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer30.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer31.norm1.weight \t torch.Size([1824])\n",
      "features.denseblock3.denselayer31.norm1.bias \t torch.Size([1824])\n",
      "features.denseblock3.denselayer31.norm1.running_mean \t torch.Size([1824])\n",
      "features.denseblock3.denselayer31.norm1.running_var \t torch.Size([1824])\n",
      "features.denseblock3.denselayer31.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer31.conv1.weight \t torch.Size([192, 1824, 1, 1])\n",
      "features.denseblock3.denselayer31.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer31.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer31.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer31.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer31.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer31.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer32.norm1.weight \t torch.Size([1872])\n",
      "features.denseblock3.denselayer32.norm1.bias \t torch.Size([1872])\n",
      "features.denseblock3.denselayer32.norm1.running_mean \t torch.Size([1872])\n",
      "features.denseblock3.denselayer32.norm1.running_var \t torch.Size([1872])\n",
      "features.denseblock3.denselayer32.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer32.conv1.weight \t torch.Size([192, 1872, 1, 1])\n",
      "features.denseblock3.denselayer32.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer32.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer32.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer32.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer32.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer32.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer33.norm1.weight \t torch.Size([1920])\n",
      "features.denseblock3.denselayer33.norm1.bias \t torch.Size([1920])\n",
      "features.denseblock3.denselayer33.norm1.running_mean \t torch.Size([1920])\n",
      "features.denseblock3.denselayer33.norm1.running_var \t torch.Size([1920])\n",
      "features.denseblock3.denselayer33.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer33.conv1.weight \t torch.Size([192, 1920, 1, 1])\n",
      "features.denseblock3.denselayer33.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer33.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer33.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer33.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer33.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer33.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer34.norm1.weight \t torch.Size([1968])\n",
      "features.denseblock3.denselayer34.norm1.bias \t torch.Size([1968])\n",
      "features.denseblock3.denselayer34.norm1.running_mean \t torch.Size([1968])\n",
      "features.denseblock3.denselayer34.norm1.running_var \t torch.Size([1968])\n",
      "features.denseblock3.denselayer34.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer34.conv1.weight \t torch.Size([192, 1968, 1, 1])\n",
      "features.denseblock3.denselayer34.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer34.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer34.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer34.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer34.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer34.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer35.norm1.weight \t torch.Size([2016])\n",
      "features.denseblock3.denselayer35.norm1.bias \t torch.Size([2016])\n",
      "features.denseblock3.denselayer35.norm1.running_mean \t torch.Size([2016])\n",
      "features.denseblock3.denselayer35.norm1.running_var \t torch.Size([2016])\n",
      "features.denseblock3.denselayer35.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer35.conv1.weight \t torch.Size([192, 2016, 1, 1])\n",
      "features.denseblock3.denselayer35.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer35.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer35.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer35.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer35.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer35.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock3.denselayer36.norm1.weight \t torch.Size([2064])\n",
      "features.denseblock3.denselayer36.norm1.bias \t torch.Size([2064])\n",
      "features.denseblock3.denselayer36.norm1.running_mean \t torch.Size([2064])\n",
      "features.denseblock3.denselayer36.norm1.running_var \t torch.Size([2064])\n",
      "features.denseblock3.denselayer36.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer36.conv1.weight \t torch.Size([192, 2064, 1, 1])\n",
      "features.denseblock3.denselayer36.norm2.weight \t torch.Size([192])\n",
      "features.denseblock3.denselayer36.norm2.bias \t torch.Size([192])\n",
      "features.denseblock3.denselayer36.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock3.denselayer36.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock3.denselayer36.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock3.denselayer36.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.transition3.norm.weight \t torch.Size([2112])\n",
      "features.transition3.norm.bias \t torch.Size([2112])\n",
      "features.transition3.norm.running_mean \t torch.Size([2112])\n",
      "features.transition3.norm.running_var \t torch.Size([2112])\n",
      "features.transition3.norm.num_batches_tracked \t torch.Size([])\n",
      "features.transition3.conv.weight \t torch.Size([1056, 2112, 1, 1])\n",
      "features.denseblock4.denselayer1.norm1.weight \t torch.Size([1056])\n",
      "features.denseblock4.denselayer1.norm1.bias \t torch.Size([1056])\n",
      "features.denseblock4.denselayer1.norm1.running_mean \t torch.Size([1056])\n",
      "features.denseblock4.denselayer1.norm1.running_var \t torch.Size([1056])\n",
      "features.denseblock4.denselayer1.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer1.conv1.weight \t torch.Size([192, 1056, 1, 1])\n",
      "features.denseblock4.denselayer1.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer1.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer1.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer1.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer1.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer1.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer2.norm1.weight \t torch.Size([1104])\n",
      "features.denseblock4.denselayer2.norm1.bias \t torch.Size([1104])\n",
      "features.denseblock4.denselayer2.norm1.running_mean \t torch.Size([1104])\n",
      "features.denseblock4.denselayer2.norm1.running_var \t torch.Size([1104])\n",
      "features.denseblock4.denselayer2.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer2.conv1.weight \t torch.Size([192, 1104, 1, 1])\n",
      "features.denseblock4.denselayer2.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer2.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer2.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer2.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer2.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer2.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer3.norm1.weight \t torch.Size([1152])\n",
      "features.denseblock4.denselayer3.norm1.bias \t torch.Size([1152])\n",
      "features.denseblock4.denselayer3.norm1.running_mean \t torch.Size([1152])\n",
      "features.denseblock4.denselayer3.norm1.running_var \t torch.Size([1152])\n",
      "features.denseblock4.denselayer3.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer3.conv1.weight \t torch.Size([192, 1152, 1, 1])\n",
      "features.denseblock4.denselayer3.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer3.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer3.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer3.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer3.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer3.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer4.norm1.weight \t torch.Size([1200])\n",
      "features.denseblock4.denselayer4.norm1.bias \t torch.Size([1200])\n",
      "features.denseblock4.denselayer4.norm1.running_mean \t torch.Size([1200])\n",
      "features.denseblock4.denselayer4.norm1.running_var \t torch.Size([1200])\n",
      "features.denseblock4.denselayer4.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer4.conv1.weight \t torch.Size([192, 1200, 1, 1])\n",
      "features.denseblock4.denselayer4.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer4.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer4.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer4.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer4.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer4.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer5.norm1.weight \t torch.Size([1248])\n",
      "features.denseblock4.denselayer5.norm1.bias \t torch.Size([1248])\n",
      "features.denseblock4.denselayer5.norm1.running_mean \t torch.Size([1248])\n",
      "features.denseblock4.denselayer5.norm1.running_var \t torch.Size([1248])\n",
      "features.denseblock4.denselayer5.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer5.conv1.weight \t torch.Size([192, 1248, 1, 1])\n",
      "features.denseblock4.denselayer5.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer5.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer5.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer5.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer5.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer5.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer6.norm1.weight \t torch.Size([1296])\n",
      "features.denseblock4.denselayer6.norm1.bias \t torch.Size([1296])\n",
      "features.denseblock4.denselayer6.norm1.running_mean \t torch.Size([1296])\n",
      "features.denseblock4.denselayer6.norm1.running_var \t torch.Size([1296])\n",
      "features.denseblock4.denselayer6.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer6.conv1.weight \t torch.Size([192, 1296, 1, 1])\n",
      "features.denseblock4.denselayer6.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer6.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer6.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer6.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer6.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer6.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer7.norm1.weight \t torch.Size([1344])\n",
      "features.denseblock4.denselayer7.norm1.bias \t torch.Size([1344])\n",
      "features.denseblock4.denselayer7.norm1.running_mean \t torch.Size([1344])\n",
      "features.denseblock4.denselayer7.norm1.running_var \t torch.Size([1344])\n",
      "features.denseblock4.denselayer7.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer7.conv1.weight \t torch.Size([192, 1344, 1, 1])\n",
      "features.denseblock4.denselayer7.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer7.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer7.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer7.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer7.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer7.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer8.norm1.weight \t torch.Size([1392])\n",
      "features.denseblock4.denselayer8.norm1.bias \t torch.Size([1392])\n",
      "features.denseblock4.denselayer8.norm1.running_mean \t torch.Size([1392])\n",
      "features.denseblock4.denselayer8.norm1.running_var \t torch.Size([1392])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features.denseblock4.denselayer8.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer8.conv1.weight \t torch.Size([192, 1392, 1, 1])\n",
      "features.denseblock4.denselayer8.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer8.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer8.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer8.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer8.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer8.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer9.norm1.weight \t torch.Size([1440])\n",
      "features.denseblock4.denselayer9.norm1.bias \t torch.Size([1440])\n",
      "features.denseblock4.denselayer9.norm1.running_mean \t torch.Size([1440])\n",
      "features.denseblock4.denselayer9.norm1.running_var \t torch.Size([1440])\n",
      "features.denseblock4.denselayer9.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer9.conv1.weight \t torch.Size([192, 1440, 1, 1])\n",
      "features.denseblock4.denselayer9.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer9.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer9.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer9.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer9.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer9.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer10.norm1.weight \t torch.Size([1488])\n",
      "features.denseblock4.denselayer10.norm1.bias \t torch.Size([1488])\n",
      "features.denseblock4.denselayer10.norm1.running_mean \t torch.Size([1488])\n",
      "features.denseblock4.denselayer10.norm1.running_var \t torch.Size([1488])\n",
      "features.denseblock4.denselayer10.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer10.conv1.weight \t torch.Size([192, 1488, 1, 1])\n",
      "features.denseblock4.denselayer10.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer10.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer10.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer10.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer10.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer10.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer11.norm1.weight \t torch.Size([1536])\n",
      "features.denseblock4.denselayer11.norm1.bias \t torch.Size([1536])\n",
      "features.denseblock4.denselayer11.norm1.running_mean \t torch.Size([1536])\n",
      "features.denseblock4.denselayer11.norm1.running_var \t torch.Size([1536])\n",
      "features.denseblock4.denselayer11.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer11.conv1.weight \t torch.Size([192, 1536, 1, 1])\n",
      "features.denseblock4.denselayer11.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer11.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer11.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer11.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer11.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer11.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer12.norm1.weight \t torch.Size([1584])\n",
      "features.denseblock4.denselayer12.norm1.bias \t torch.Size([1584])\n",
      "features.denseblock4.denselayer12.norm1.running_mean \t torch.Size([1584])\n",
      "features.denseblock4.denselayer12.norm1.running_var \t torch.Size([1584])\n",
      "features.denseblock4.denselayer12.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer12.conv1.weight \t torch.Size([192, 1584, 1, 1])\n",
      "features.denseblock4.denselayer12.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer12.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer12.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer12.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer12.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer12.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer13.norm1.weight \t torch.Size([1632])\n",
      "features.denseblock4.denselayer13.norm1.bias \t torch.Size([1632])\n",
      "features.denseblock4.denselayer13.norm1.running_mean \t torch.Size([1632])\n",
      "features.denseblock4.denselayer13.norm1.running_var \t torch.Size([1632])\n",
      "features.denseblock4.denselayer13.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer13.conv1.weight \t torch.Size([192, 1632, 1, 1])\n",
      "features.denseblock4.denselayer13.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer13.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer13.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer13.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer13.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer13.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer14.norm1.weight \t torch.Size([1680])\n",
      "features.denseblock4.denselayer14.norm1.bias \t torch.Size([1680])\n",
      "features.denseblock4.denselayer14.norm1.running_mean \t torch.Size([1680])\n",
      "features.denseblock4.denselayer14.norm1.running_var \t torch.Size([1680])\n",
      "features.denseblock4.denselayer14.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer14.conv1.weight \t torch.Size([192, 1680, 1, 1])\n",
      "features.denseblock4.denselayer14.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer14.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer14.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer14.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer14.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer14.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer15.norm1.weight \t torch.Size([1728])\n",
      "features.denseblock4.denselayer15.norm1.bias \t torch.Size([1728])\n",
      "features.denseblock4.denselayer15.norm1.running_mean \t torch.Size([1728])\n",
      "features.denseblock4.denselayer15.norm1.running_var \t torch.Size([1728])\n",
      "features.denseblock4.denselayer15.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer15.conv1.weight \t torch.Size([192, 1728, 1, 1])\n",
      "features.denseblock4.denselayer15.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer15.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer15.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer15.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer15.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer15.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer16.norm1.weight \t torch.Size([1776])\n",
      "features.denseblock4.denselayer16.norm1.bias \t torch.Size([1776])\n",
      "features.denseblock4.denselayer16.norm1.running_mean \t torch.Size([1776])\n",
      "features.denseblock4.denselayer16.norm1.running_var \t torch.Size([1776])\n",
      "features.denseblock4.denselayer16.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer16.conv1.weight \t torch.Size([192, 1776, 1, 1])\n",
      "features.denseblock4.denselayer16.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer16.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer16.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer16.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer16.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer16.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer17.norm1.weight \t torch.Size([1824])\n",
      "features.denseblock4.denselayer17.norm1.bias \t torch.Size([1824])\n",
      "features.denseblock4.denselayer17.norm1.running_mean \t torch.Size([1824])\n",
      "features.denseblock4.denselayer17.norm1.running_var \t torch.Size([1824])\n",
      "features.denseblock4.denselayer17.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer17.conv1.weight \t torch.Size([192, 1824, 1, 1])\n",
      "features.denseblock4.denselayer17.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer17.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer17.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer17.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer17.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer17.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer18.norm1.weight \t torch.Size([1872])\n",
      "features.denseblock4.denselayer18.norm1.bias \t torch.Size([1872])\n",
      "features.denseblock4.denselayer18.norm1.running_mean \t torch.Size([1872])\n",
      "features.denseblock4.denselayer18.norm1.running_var \t torch.Size([1872])\n",
      "features.denseblock4.denselayer18.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer18.conv1.weight \t torch.Size([192, 1872, 1, 1])\n",
      "features.denseblock4.denselayer18.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer18.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer18.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer18.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer18.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer18.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer19.norm1.weight \t torch.Size([1920])\n",
      "features.denseblock4.denselayer19.norm1.bias \t torch.Size([1920])\n",
      "features.denseblock4.denselayer19.norm1.running_mean \t torch.Size([1920])\n",
      "features.denseblock4.denselayer19.norm1.running_var \t torch.Size([1920])\n",
      "features.denseblock4.denselayer19.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer19.conv1.weight \t torch.Size([192, 1920, 1, 1])\n",
      "features.denseblock4.denselayer19.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer19.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer19.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer19.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer19.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer19.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer20.norm1.weight \t torch.Size([1968])\n",
      "features.denseblock4.denselayer20.norm1.bias \t torch.Size([1968])\n",
      "features.denseblock4.denselayer20.norm1.running_mean \t torch.Size([1968])\n",
      "features.denseblock4.denselayer20.norm1.running_var \t torch.Size([1968])\n",
      "features.denseblock4.denselayer20.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer20.conv1.weight \t torch.Size([192, 1968, 1, 1])\n",
      "features.denseblock4.denselayer20.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer20.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer20.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer20.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer20.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer20.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer21.norm1.weight \t torch.Size([2016])\n",
      "features.denseblock4.denselayer21.norm1.bias \t torch.Size([2016])\n",
      "features.denseblock4.denselayer21.norm1.running_mean \t torch.Size([2016])\n",
      "features.denseblock4.denselayer21.norm1.running_var \t torch.Size([2016])\n",
      "features.denseblock4.denselayer21.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer21.conv1.weight \t torch.Size([192, 2016, 1, 1])\n",
      "features.denseblock4.denselayer21.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer21.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer21.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer21.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer21.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer21.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer22.norm1.weight \t torch.Size([2064])\n",
      "features.denseblock4.denselayer22.norm1.bias \t torch.Size([2064])\n",
      "features.denseblock4.denselayer22.norm1.running_mean \t torch.Size([2064])\n",
      "features.denseblock4.denselayer22.norm1.running_var \t torch.Size([2064])\n",
      "features.denseblock4.denselayer22.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer22.conv1.weight \t torch.Size([192, 2064, 1, 1])\n",
      "features.denseblock4.denselayer22.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer22.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer22.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer22.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer22.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer22.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer23.norm1.weight \t torch.Size([2112])\n",
      "features.denseblock4.denselayer23.norm1.bias \t torch.Size([2112])\n",
      "features.denseblock4.denselayer23.norm1.running_mean \t torch.Size([2112])\n",
      "features.denseblock4.denselayer23.norm1.running_var \t torch.Size([2112])\n",
      "features.denseblock4.denselayer23.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer23.conv1.weight \t torch.Size([192, 2112, 1, 1])\n",
      "features.denseblock4.denselayer23.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer23.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer23.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer23.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer23.norm2.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer23.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.denseblock4.denselayer24.norm1.weight \t torch.Size([2160])\n",
      "features.denseblock4.denselayer24.norm1.bias \t torch.Size([2160])\n",
      "features.denseblock4.denselayer24.norm1.running_mean \t torch.Size([2160])\n",
      "features.denseblock4.denselayer24.norm1.running_var \t torch.Size([2160])\n",
      "features.denseblock4.denselayer24.norm1.num_batches_tracked \t torch.Size([])\n",
      "features.denseblock4.denselayer24.conv1.weight \t torch.Size([192, 2160, 1, 1])\n",
      "features.denseblock4.denselayer24.norm2.weight \t torch.Size([192])\n",
      "features.denseblock4.denselayer24.norm2.bias \t torch.Size([192])\n",
      "features.denseblock4.denselayer24.norm2.running_mean \t torch.Size([192])\n",
      "features.denseblock4.denselayer24.norm2.running_var \t torch.Size([192])\n",
      "features.denseblock4.denselayer24.norm2.num_batches_tracked \t torch.Size([])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features.denseblock4.denselayer24.conv2.weight \t torch.Size([48, 192, 3, 3])\n",
      "features.norm5.weight \t torch.Size([2208])\n",
      "features.norm5.bias \t torch.Size([2208])\n",
      "features.norm5.running_mean \t torch.Size([2208])\n",
      "features.norm5.running_var \t torch.Size([2208])\n",
      "features.norm5.num_batches_tracked \t torch.Size([])\n",
      "classifier.weight \t torch.Size([2, 2208])\n",
      "classifier.bias \t torch.Size([2])\n",
      "Optimizer's state_dict:\n",
      "state \t {}\n",
      "param_groups \t [{'lr': 0.001, 'betas': (0.9, 0.999), 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False, 'params': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483]}]\n"
     ]
    }
   ],
   "source": [
    "# Print model's state_dict\n",
    "print(\"Model's state_dict:\")\n",
    "for param_tensor in model_loaded.state_dict():\n",
    "    print(param_tensor, \"\\t\", model_loaded.state_dict()[param_tensor].size())\n",
    "\n",
    "# Print optimizer's state_dict\n",
    "print(\"Optimizer's state_dict:\")\n",
    "for var_name in optimizer_loaded.state_dict():\n",
    "    print(var_name, \"\\t\", optimizer_loaded.state_dict()[var_name])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Initialize parameters for second phase of training (optional)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learning parameters\n",
    "batch_size = 64\n",
    "new_epochs = 1\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "# train for more epochs\n",
    "epochs = new_epochs\n",
    "print(f\"Train for {epochs} more epochs...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history_loaded = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    epoch_start = time.time()\n",
    "    print(\"Epoch: {}/{}\".format(epoch, epochs))\n",
    "     \n",
    "    # Set gradient calculation to ON. Needed during training.\n",
    "    torch.set_grad_enabled(True)\n",
    "        \n",
    "    # Set to training mode\n",
    "    model_loaded.train()\n",
    "     \n",
    "    # Loss and Accuracy within the epoch\n",
    "    train_loss = 0.0\n",
    "    train_acc = 0.0\n",
    "     \n",
    "    valid_loss = 0.0\n",
    "    valid_acc = 0.0\n",
    " \n",
    "    # Iterate through all batches of training data\n",
    "    for i, (inputs, labels) in enumerate(train_data):\n",
    " \n",
    "        inputs = inputs.to(device)\n",
    "        labels = labels.to(device)\n",
    "         \n",
    "        # Clean existing gradients\n",
    "        optimizer_loaded.zero_grad()\n",
    "         \n",
    "        # Forward pass - compute outputs on input data using the model\n",
    "        outputs = model_loaded(inputs)\n",
    "        \n",
    "        # Compute loss\n",
    "        loss = criterion_loaded(outputs, labels)\n",
    "         \n",
    "        # Backpropagate the gradients\n",
    "        loss.backward()\n",
    "         \n",
    "        # Update the parameters\n",
    "        optimizer_loaded.step()\n",
    "         \n",
    "        # Compute the total loss for the batch and add it to train_loss\n",
    "        train_loss += loss.item() * inputs.size(0)\n",
    "         \n",
    "        # Compute the accuracy\n",
    "        ret, predictions = torch.max(outputs.data, 1)\n",
    "\n",
    "        correct_counts = predictions.eq(labels.data.view_as(predictions))\n",
    "         \n",
    "        # Convert correct_counts to float and then compute the mean\n",
    "        acc = torch.mean(correct_counts.type(torch.FloatTensor))\n",
    "         \n",
    "        # Compute total accuracy in the whole batch and add to train_acc\n",
    "        train_acc += acc.item() * inputs.size(0)\n",
    "\n",
    "         \n",
    "        print(\"Batch number: {:03d}, Training: Loss: {:.4f}, Accuracy: {:.4f}\".format(i, loss.item(), acc.item()))\n",
    "        \n",
    "\n",
    "    # Validation is carried out in each epoch immediately after the training loop\n",
    "    # Validation - No gradient calculation is needed\n",
    "    with torch.no_grad():\n",
    "\n",
    "        # Set to evaluation mode\n",
    "        model_loaded.eval()\n",
    "\n",
    "        # Validation loop\n",
    "        # Iterate through all batches of validation data\n",
    "        for j, (inputs, labels) in enumerate(valid_data):\n",
    "            inputs = inputs.to(device)\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            # Forward pass - compute outputs on input data using the model\n",
    "            outputs = model_loaded(inputs)\n",
    "\n",
    "            # Compute loss\n",
    "            loss = criterion_loaded(outputs, labels)\n",
    "\n",
    "            # Compute the total loss for the batch and add it to valid_loss\n",
    "            valid_loss += loss.item() * inputs.size(0)\n",
    "\n",
    "            # Calculate validation accuracy\n",
    "            ret, predictions = torch.max(outputs.data, 1)\n",
    "            correct_counts = predictions.eq(labels.data.view_as(predictions))\n",
    "\n",
    "            # Convert correct_counts to float and then compute the mean\n",
    "            acc = torch.mean(correct_counts.type(torch.FloatTensor))\n",
    "\n",
    "            # Compute total accuracy in the whole batch and add to valid_acc\n",
    "            valid_acc += acc.item() * inputs.size(0)\n",
    "\n",
    "\n",
    "            print(\"Validation Batch number: {:03d}, Validation: Loss: {:.4f}, Accuracy: {:.4f}\".format(j, loss.item(), acc.item()))\n",
    "\n",
    "    # Find average training loss and training accuracy\n",
    "    avg_train_loss = train_loss/train_data_size\n",
    "    avg_train_acc = train_acc/float(train_data_size)\n",
    "\n",
    "    # Find average training loss and training accuracy\n",
    "    avg_valid_loss = valid_loss/valid_data_size\n",
    "    avg_valid_acc = valid_acc/float(valid_data_size)\n",
    "\n",
    "    history_loaded.append([avg_train_loss, avg_valid_loss, avg_train_acc, avg_valid_acc])\n",
    "\n",
    "    epoch_end = time.time()\n",
    "\n",
    "    print(\"Epoch : {:03d}, Training: Loss: {:.4f}, Accuracy: {:.4f}%, \\n\\t\\tValidation : Loss : {:.4f}, Accuracy: {:.4f}%, \\n Time (train+val): {:.4f}s\".format(epoch, avg_train_loss, avg_train_acc*100, avg_valid_loss, avg_valid_acc*100, epoch_end-epoch_start))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check if the current model was loaded, or it was just trained. \n",
    "# Needed for compatibility with testing stage.\n",
    "if model_loaded: \n",
    "    model = model_loaded\n",
    "    criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Evaluate model based on test dataset**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy = 0.8409658150817465\n",
      "precision = 0.8967206558688262\n",
      "recall = 0.6591460277798192\n",
      "f1 = 0.759794993434707\n",
      "Test \n",
      "Loss : 0.3794, Accuracy: 84.0966%, \n",
      "Time : 5728.6465s\n"
     ]
    }
   ],
   "source": [
    "test_start = time.time()\n",
    "\n",
    "# Loss and Accuracy within the epoch\n",
    "test_loss = 0.0\n",
    "test_acc = 0.0\n",
    "\n",
    "# Initialize an empty tensor. This will store all the predictions and will be used for metrics.\n",
    "tot_predictions = torch.Tensor()\n",
    "# Initialize an empty tensor. This will store all the ground truth labels and will be used for metrics.\n",
    "tot_labels = torch.Tensor()\n",
    "\n",
    "# Testing - No gradient calculation is needed\n",
    "with torch.no_grad():\n",
    "\n",
    "    # Set to evaluation mode\n",
    "    model.eval()\n",
    "\n",
    "    # Validation loop\n",
    "    # Iterate through all batches of test data\n",
    "    for j, (inputs, labels) in enumerate(test_data):\n",
    "        inputs = inputs.to(device)\n",
    "        labels = labels.to(device)\n",
    "        \n",
    "        # Add values to ground truth\n",
    "        tot_labels = torch.cat((tot_labels, labels))\n",
    "\n",
    "        # Forward pass - compute outputs on input data using the model\n",
    "        outputs = model(inputs)\n",
    "\n",
    "        # Compute loss\n",
    "        loss = criterion(outputs, labels)\n",
    "\n",
    "        # Compute the total loss for the batch and add it to test_loss\n",
    "        test_loss += loss.item() * inputs.size(0)\n",
    "\n",
    "        # Calculate validation accuracy\n",
    "        ret, predictions = torch.max(outputs.data, 1)\n",
    "        correct_counts = predictions.eq(labels.data.view_as(predictions))\n",
    "        tot_predictions = torch.cat((tot_predictions, predictions))\n",
    "\n",
    "        # Convert correct_counts to float and then compute the mean\n",
    "        acc = torch.mean(correct_counts.type(torch.FloatTensor))\n",
    "\n",
    "        # Compute total accuracy in the whole batch and add to test_acc\n",
    "        test_acc += acc.item() * inputs.size(0)\n",
    "\n",
    "#             print(\"Validation Batch number: {:03d}, Validation: Loss: {:.4f}, Accuracy: {:.4f}\".format(j, loss.item(), acc.item()))\n",
    "\n",
    "\n",
    "# Find average testing loss and accuracy\n",
    "avg_test_loss = test_loss/test_data_size\n",
    "avg_test_acc = test_acc/float(test_data_size)\n",
    "\n",
    "accuracy = metrics.accuracy_score(tot_labels.to(device), tot_predictions.to(device))\n",
    "\n",
    "# The precision is intuitively the ability of the classifier not to label as positive a sample that is negative.\n",
    "precision = metrics.precision_score(tot_labels.to(device), tot_predictions.to(device), average='binary')\n",
    "\n",
    "# The recall is intuitively the ability of the classifier to find all the positive samples.\n",
    "recall = metrics.recall_score(tot_labels.to(device), tot_predictions.to(device), pos_label=1, average='binary')\n",
    "\n",
    "# The F1 score can be interpreted as a weighted average of the precision and recall\n",
    "f1 = metrics.f1_score(tot_labels.to(device), tot_predictions.to(device), average='binary')\n",
    "\n",
    "# history.append([avg_train_loss, avg_valid_loss, avg_train_acc, avg_valid_acc])\n",
    "\n",
    "test_end = time.time()\n",
    "print(f\"accuracy = {accuracy}\")\n",
    "print(f\"precision = {precision}\")\n",
    "print(f\"recall = {recall}\")\n",
    "print(f\"f1 = {f1}\")\n",
    "print(\"Test \\nLoss : {:.4f}, Accuracy: {:.4f}%, \\nTime : {:.4f}s\".format(avg_test_loss, avg_test_acc*100, test_end-test_start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}