Resnet_20_Non_Private.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from torch.autograd import Variable\n",
    "from opacus import PrivacyEngine\n",
    "from opacus.validators import ModuleValidator\n",
    "from opacus.utils.batch_memory_manager import BatchMemoryManager\n",
    "import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPSILON = 3.0\n",
    "DELTA = 1e-5\n",
    "MAX_GRAD_NORM = 1.5 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Device configuration\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# Image preprocessing modules\n",
    "normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n",
    "\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.Pad(4),\n",
    "    transforms.RandomRotation(10),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.RandomCrop(32),\n",
    "    transforms.ToTensor(),\n",
    "    normalize])\n",
    "    \n",
    "test_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    normalize])\n",
    "\n",
    "train_dataset = torchvision.datasets.CIFAR10(root='data/',train=True, transform=train_transform,download=True)\n",
    "test_dataset = torchvision.datasets.CIFAR10(root='data/',train=False, transform=test_transform)\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,batch_size=64, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,batch_size=64, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyper-parameters\n",
    "num_epochs = 40\n",
    "learning_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For updating learning rate\n",
    "def update_lr(optimizer, lr):\n",
    "    \"\"\"\n",
    "    This method update learning rate\n",
    "    \"\"\"\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv3x3(in_channels, out_channels, stride=1):\n",
    "    \"\"\"\n",
    "    return 3x3 Conv2d\n",
    "    \"\"\"\n",
    "    return nn.Conv2d(in_channels, out_channels, kernel_size=3,stride=stride, padding=1, bias=False)\n",
    "\n",
    "\n",
    "class ResidualBlock(nn.Module):\n",
    "    \"\"\"\n",
    "    Initialize basic ResidualBlock with forward propogation\n",
    "    \"\"\"\n",
    "    def __init__(self, in_channels, out_channels, stride=1, downsample=None):\n",
    "        super(ResidualBlock, self).__init__()\n",
    "        self.conv1 = conv3x3(in_channels, out_channels, stride)\n",
    "        self.bn1 = nn.BatchNorm2d(out_channels)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.conv2 = conv3x3(out_channels, out_channels)\n",
    "        self.bn2 = nn.BatchNorm2d(out_channels)\n",
    "        self.downsample = downsample\n",
    "\n",
    "    def forward(self, x):\n",
    "        residual = x\n",
    "        out = self.conv1(x)\n",
    "        out = self.bn1(out)\n",
    "        out = self.relu(out)\n",
    "        out = self.conv2(out)\n",
    "        out = self.bn2(out)\n",
    "        if self.downsample:\n",
    "            residual = self.downsample(x)\n",
    "        out += residual\n",
    "        out = self.relu(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResNet(nn.Module):\n",
    "    \"\"\"\n",
    "    Initialize  ResNet with forward propogation\n",
    "    \"\"\"\n",
    "    def __init__(self, block, layers, num_classes=10):\n",
    "        super(ResNet, self).__init__()\n",
    "        self.in_channels = 16\n",
    "        self.conv = conv3x3(3, 16)\n",
    "        self.bn = nn.BatchNorm2d(16)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.layer1 = self.make_layer(block, 16, layers[0])\n",
    "        self.layer2 = self.make_layer(block, 32, layers[1], 2)\n",
    "        self.layer3 = self.make_layer(block, 64, layers[2], 2)\n",
    "        self.avg_pool = nn.AvgPool2d(8)\n",
    "        self.fc = nn.Linear(64, num_classes)\n",
    "\n",
    "    def make_layer(self, block, out_channels, blocks, stride=1):\n",
    "        downsample = None\n",
    "        if (stride != 1) or (self.in_channels != out_channels):\n",
    "            downsample = nn.Sequential(\n",
    "                conv3x3(self.in_channels, out_channels, stride=stride),\n",
    "                nn.BatchNorm2d(out_channels))\n",
    "        layers = []\n",
    "        layers.append(block(self.in_channels, out_channels, stride, downsample))\n",
    "        self.in_channels = out_channels\n",
    "        for i in range(1, blocks):\n",
    "            layers.append(block(out_channels, out_channels))\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = self.conv(x)\n",
    "        out = self.bn(out)\n",
    "        out = self.relu(out)\n",
    "        out = self.layer1(out)\n",
    "        out = self.layer2(out)\n",
    "        out = self.layer3(out)\n",
    "        out = self.avg_pool(out)\n",
    "        out = out.view(out.size(0), -1)\n",
    "        out = self.fc(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ResNet(ResidualBlock, [3, 3, 3]).to(device)\n",
    "model.eval();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "errors = ModuleValidator.validate(model, strict=False)\n",
    "if errors:  # If there are errors, fix them\n",
    "    model = ModuleValidator.fix(model)\n",
    "ModuleValidator.validate(model, strict=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loss and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "privacy_engine = PrivacyEngine()\n",
    "\n",
    "model, optimizer, train_loader = privacy_engine.make_private_with_epsilon(\n",
    "    module=model,\n",
    "    optimizer=optimizer,\n",
    "    data_loader=train_loader,\n",
    "    epochs=num_epochs,\n",
    "    target_epsilon=EPSILON,\n",
    "    target_delta=DELTA,\n",
    "    max_grad_norm=MAX_GRAD_NORM,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total parameters: 292954\n",
      "Trainable parameters: 292954\n"
     ]
    }
   ],
   "source": [
    "total_params = sum(p.numel() for p in model.parameters())\n",
    "\n",
    "# To count only the trainable parameters\n",
    "trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n",
    "print(f\"Total parameters: {total_params}\")\n",
    "print(f\"Trainable parameters: {trainable_params}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model):\n",
    "    \"\"\"\n",
    "    Evaluate accuracy of test set and save weight of model\n",
    "    \"\"\"\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        correct = 0\n",
    "        total = 0\n",
    "        for images, labels in test_loader:\n",
    "            images = images.to(device)\n",
    "            labels = labels.to(device)\n",
    "            outputs = model(images)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            total += labels.size(0)\n",
    "            correct += (predicted == labels).sum().item()\n",
    "\n",
    "        print('Accuracy of the model on the test images: {} %'.format(100 * correct / total))\n",
    "\n",
    "    # Save the model checkpoint\n",
    "    torch.save(model.state_dict(), 'model_weight/'+str(int(100 * correct / total))+'resnet.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/40], Step [100/782] Loss: 2.3002\n",
      "Epoch [1/40], Step [200/782] Loss: 2.2941\n",
      "Epoch [1/40], Step [300/782] Loss: 2.3043\n",
      "Epoch [1/40], Step [400/782] Loss: 2.2989\n",
      "Epoch [1/40], Step [500/782] Loss: 2.3021\n",
      "Epoch [1/40], Step [600/782] Loss: 2.3072\n",
      "Epoch [1/40], Step [700/782] Loss: 2.3143\n",
      "Epoch [2/40], Step [100/782] Loss: 2.2934\n",
      "Epoch [2/40], Step [200/782] Loss: 2.2956\n",
      "Epoch [2/40], Step [300/782] Loss: 2.2991\n",
      "Epoch [2/40], Step [400/782] Loss: 2.2870\n",
      "Epoch [2/40], Step [500/782] Loss: 2.3026\n",
      "Epoch [2/40], Step [600/782] Loss: 2.3087\n",
      "Epoch [2/40], Step [700/782] Loss: 2.3012\n",
      "Epoch [3/40], Step [100/782] Loss: 2.3049\n",
      "Epoch [3/40], Step [200/782] Loss: 2.2908\n",
      "Epoch [3/40], Step [300/782] Loss: 2.3103\n",
      "Epoch [3/40], Step [400/782] Loss: 2.3003\n",
      "Epoch [3/40], Step [500/782] Loss: 2.2877\n",
      "Epoch [3/40], Step [600/782] Loss: 2.2924\n",
      "Epoch [3/40], Step [700/782] Loss: 2.2862\n",
      "Epoch [4/40], Step [100/782] Loss: 2.2841\n",
      "Epoch [4/40], Step [200/782] Loss: 2.2608\n",
      "Epoch [4/40], Step [300/782] Loss: 2.2761\n",
      "Epoch [4/40], Step [400/782] Loss: 2.2865\n",
      "Epoch [4/40], Step [500/782] Loss: 2.2651\n",
      "Epoch [4/40], Step [600/782] Loss: 2.2622\n",
      "Epoch [4/40], Step [700/782] Loss: 2.3188\n",
      "Epoch [5/40], Step [100/782] Loss: 2.2910\n",
      "Epoch [5/40], Step [200/782] Loss: 2.2352\n",
      "Epoch [5/40], Step [300/782] Loss: 2.2356\n",
      "Epoch [5/40], Step [400/782] Loss: 2.2259\n",
      "Epoch [5/40], Step [500/782] Loss: 2.2276\n",
      "Epoch [5/40], Step [600/782] Loss: 2.2584\n",
      "Epoch [5/40], Step [700/782] Loss: 2.2756\n",
      "Accuracy of the model on the test images: 13.37 %\n",
      "Epoch [6/40], Step [100/782] Loss: 2.2471\n",
      "Epoch [6/40], Step [200/782] Loss: 2.1941\n",
      "Epoch [6/40], Step [300/782] Loss: 2.2764\n",
      "Epoch [6/40], Step [400/782] Loss: 2.1752\n",
      "Epoch [6/40], Step [500/782] Loss: 2.1754\n",
      "Epoch [6/40], Step [600/782] Loss: 2.2491\n",
      "Epoch [6/40], Step [700/782] Loss: 2.2042\n",
      "Epoch [7/40], Step [100/782] Loss: 2.1735\n",
      "Epoch [7/40], Step [200/782] Loss: 2.1690\n",
      "Epoch [7/40], Step [300/782] Loss: 2.1397\n",
      "Epoch [7/40], Step [400/782] Loss: 2.2053\n",
      "Epoch [7/40], Step [500/782] Loss: 2.1178\n",
      "Epoch [7/40], Step [600/782] Loss: 2.0432\n",
      "Epoch [7/40], Step [700/782] Loss: 2.0849\n",
      "Epoch [8/40], Step [100/782] Loss: 1.9992\n",
      "Epoch [8/40], Step [200/782] Loss: 1.9865\n",
      "Epoch [8/40], Step [300/782] Loss: 2.0357\n",
      "Epoch [8/40], Step [400/782] Loss: 1.9723\n",
      "Epoch [8/40], Step [500/782] Loss: 1.9019\n",
      "Epoch [8/40], Step [600/782] Loss: 2.0616\n",
      "Epoch [8/40], Step [700/782] Loss: 2.2865\n",
      "Epoch [9/40], Step [100/782] Loss: 2.1477\n",
      "Epoch [9/40], Step [200/782] Loss: 1.9999\n",
      "Epoch [9/40], Step [300/782] Loss: 1.8369\n",
      "Epoch [9/40], Step [400/782] Loss: 2.0470\n",
      "Epoch [9/40], Step [500/782] Loss: 1.9563\n",
      "Epoch [9/40], Step [600/782] Loss: 2.0519\n",
      "Epoch [9/40], Step [700/782] Loss: 1.9694\n",
      "Epoch [10/40], Step [100/782] Loss: 1.9813\n",
      "Epoch [10/40], Step [200/782] Loss: 2.0106\n",
      "Epoch [10/40], Step [300/782] Loss: 1.8777\n",
      "Epoch [10/40], Step [400/782] Loss: 1.9138\n",
      "Epoch [10/40], Step [500/782] Loss: 1.8648\n",
      "Epoch [10/40], Step [600/782] Loss: 1.9552\n",
      "Epoch [10/40], Step [700/782] Loss: 1.8274\n",
      "Accuracy of the model on the test images: 25.29 %\n",
      "Epoch [11/40], Step [100/782] Loss: 1.7988\n",
      "Epoch [11/40], Step [200/782] Loss: 1.8948\n",
      "Epoch [11/40], Step [300/782] Loss: 1.8050\n",
      "Epoch [11/40], Step [400/782] Loss: 1.9377\n",
      "Epoch [11/40], Step [500/782] Loss: 2.1324\n",
      "Epoch [11/40], Step [600/782] Loss: 2.0958\n",
      "Epoch [11/40], Step [700/782] Loss: 1.7957\n",
      "Epoch [12/40], Step [100/782] Loss: 1.9635\n",
      "Epoch [12/40], Step [200/782] Loss: 1.8825\n",
      "Epoch [12/40], Step [300/782] Loss: 1.6944\n",
      "Epoch [12/40], Step [400/782] Loss: 1.6972\n",
      "Epoch [12/40], Step [500/782] Loss: 1.7948\n",
      "Epoch [12/40], Step [600/782] Loss: 1.8206\n",
      "Epoch [12/40], Step [700/782] Loss: 1.7446\n",
      "Epoch [13/40], Step [100/782] Loss: 1.7279\n",
      "Epoch [13/40], Step [200/782] Loss: 1.7440\n",
      "Epoch [13/40], Step [300/782] Loss: 1.9152\n",
      "Epoch [13/40], Step [400/782] Loss: 1.7785\n",
      "Epoch [13/40], Step [500/782] Loss: 1.6047\n",
      "Epoch [13/40], Step [600/782] Loss: 1.8058\n",
      "Epoch [13/40], Step [700/782] Loss: 1.7412\n",
      "Epoch [14/40], Step [100/782] Loss: 1.8145\n",
      "Epoch [14/40], Step [200/782] Loss: 1.6851\n",
      "Epoch [14/40], Step [300/782] Loss: 1.6565\n",
      "Epoch [14/40], Step [400/782] Loss: 1.7101\n",
      "Epoch [14/40], Step [500/782] Loss: 1.7033\n",
      "Epoch [14/40], Step [600/782] Loss: 1.7106\n",
      "Epoch [14/40], Step [700/782] Loss: 1.7809\n",
      "Epoch [15/40], Step [100/782] Loss: 1.7154\n",
      "Epoch [15/40], Step [200/782] Loss: 1.6122\n",
      "Epoch [15/40], Step [300/782] Loss: 1.6846\n",
      "Epoch [15/40], Step [400/782] Loss: 2.0308\n",
      "Epoch [15/40], Step [500/782] Loss: 1.7521\n",
      "Epoch [15/40], Step [600/782] Loss: 1.6841\n",
      "Epoch [15/40], Step [700/782] Loss: 1.7656\n",
      "Accuracy of the model on the test images: 36.93 %\n",
      "Epoch [16/40], Step [100/782] Loss: 1.7060\n",
      "Epoch [16/40], Step [200/782] Loss: 1.7427\n",
      "Epoch [16/40], Step [300/782] Loss: 1.5368\n",
      "Epoch [16/40], Step [400/782] Loss: 1.6825\n",
      "Epoch [16/40], Step [500/782] Loss: 1.8996\n",
      "Epoch [16/40], Step [600/782] Loss: 1.6417\n",
      "Epoch [16/40], Step [700/782] Loss: 1.7760\n",
      "Epoch [17/40], Step [100/782] Loss: 1.8990\n",
      "Epoch [17/40], Step [200/782] Loss: 1.8270\n",
      "Epoch [17/40], Step [300/782] Loss: 1.6522\n",
      "Epoch [17/40], Step [400/782] Loss: 1.8627\n",
      "Epoch [17/40], Step [500/782] Loss: 1.6090\n",
      "Epoch [17/40], Step [600/782] Loss: 1.6559\n",
      "Epoch [17/40], Step [700/782] Loss: 1.7141\n",
      "Epoch [18/40], Step [100/782] Loss: 1.7041\n",
      "Epoch [18/40], Step [200/782] Loss: 1.7428\n",
      "Epoch [18/40], Step [300/782] Loss: 1.6862\n",
      "Epoch [18/40], Step [400/782] Loss: 1.7782\n",
      "Epoch [18/40], Step [500/782] Loss: 1.3628\n",
      "Epoch [18/40], Step [600/782] Loss: 1.6907\n",
      "Epoch [18/40], Step [700/782] Loss: 1.8986\n",
      "Epoch [19/40], Step [100/782] Loss: 1.7113\n",
      "Epoch [19/40], Step [200/782] Loss: 1.5469\n",
      "Epoch [19/40], Step [300/782] Loss: 1.6345\n",
      "Epoch [19/40], Step [400/782] Loss: 1.8259\n",
      "Epoch [19/40], Step [500/782] Loss: 1.6758\n",
      "Epoch [19/40], Step [600/782] Loss: 1.8511\n",
      "Epoch [19/40], Step [700/782] Loss: 1.6045\n",
      "Epoch [20/40], Step [100/782] Loss: 1.8291\n",
      "Epoch [20/40], Step [200/782] Loss: 1.7635\n",
      "Epoch [20/40], Step [300/782] Loss: 1.6345\n",
      "Epoch [20/40], Step [400/782] Loss: 1.6606\n",
      "Epoch [20/40], Step [500/782] Loss: 1.5634\n",
      "Epoch [20/40], Step [600/782] Loss: 1.7000\n",
      "Epoch [20/40], Step [700/782] Loss: 1.8674\n",
      "Accuracy of the model on the test images: 38.68 %\n",
      "Epoch [21/40], Step [100/782] Loss: 1.7015\n",
      "Epoch [21/40], Step [200/782] Loss: 1.6330\n",
      "Epoch [21/40], Step [300/782] Loss: 1.7757\n",
      "Epoch [21/40], Step [400/782] Loss: 1.6785\n",
      "Epoch [21/40], Step [500/782] Loss: 1.6335\n",
      "Epoch [21/40], Step [600/782] Loss: 1.6668\n",
      "Epoch [21/40], Step [700/782] Loss: 1.7773\n",
      "Epoch [22/40], Step [100/782] Loss: 1.5505\n",
      "Epoch [22/40], Step [200/782] Loss: 1.5961\n",
      "Epoch [22/40], Step [300/782] Loss: 1.5699\n",
      "Epoch [22/40], Step [400/782] Loss: 1.5727\n",
      "Epoch [22/40], Step [500/782] Loss: 1.5858\n",
      "Epoch [22/40], Step [600/782] Loss: 1.7064\n",
      "Epoch [22/40], Step [700/782] Loss: 1.7070\n",
      "Epoch [23/40], Step [100/782] Loss: 1.8765\n",
      "Epoch [23/40], Step [200/782] Loss: 1.8256\n",
      "Epoch [23/40], Step [300/782] Loss: 1.6227\n",
      "Epoch [23/40], Step [400/782] Loss: 1.6390\n",
      "Epoch [23/40], Step [500/782] Loss: 1.6921\n",
      "Epoch [23/40], Step [600/782] Loss: 1.8092\n",
      "Epoch [23/40], Step [700/782] Loss: 1.8682\n",
      "Epoch [24/40], Step [100/782] Loss: 1.5897\n",
      "Epoch [24/40], Step [200/782] Loss: 1.6259\n",
      "Epoch [24/40], Step [300/782] Loss: 1.7791\n",
      "Epoch [24/40], Step [400/782] Loss: 1.6901\n",
      "Epoch [24/40], Step [500/782] Loss: 1.7877\n",
      "Epoch [24/40], Step [600/782] Loss: 1.5815\n",
      "Epoch [24/40], Step [700/782] Loss: 1.8297\n",
      "Epoch [25/40], Step [100/782] Loss: 1.4222\n",
      "Epoch [25/40], Step [200/782] Loss: 1.5998\n",
      "Epoch [25/40], Step [300/782] Loss: 1.4794\n",
      "Epoch [25/40], Step [400/782] Loss: 1.8813\n",
      "Epoch [25/40], Step [500/782] Loss: 1.5435\n",
      "Epoch [25/40], Step [600/782] Loss: 1.7611\n",
      "Epoch [25/40], Step [700/782] Loss: 1.5837\n",
      "Accuracy of the model on the test images: 38.47 %\n",
      "Epoch [26/40], Step [100/782] Loss: 1.7283\n",
      "Epoch [26/40], Step [200/782] Loss: 1.6774\n",
      "Epoch [26/40], Step [300/782] Loss: 1.9205\n",
      "Epoch [26/40], Step [400/782] Loss: 1.5878\n",
      "Epoch [26/40], Step [500/782] Loss: 1.6259\n",
      "Epoch [26/40], Step [600/782] Loss: 1.7497\n",
      "Epoch [26/40], Step [700/782] Loss: 1.5837\n",
      "Epoch [27/40], Step [100/782] Loss: 1.6316\n",
      "Epoch [27/40], Step [200/782] Loss: 1.6828\n",
      "Epoch [27/40], Step [300/782] Loss: 1.6181\n",
      "Epoch [27/40], Step [400/782] Loss: 1.7274\n",
      "Epoch [27/40], Step [500/782] Loss: 1.6878\n",
      "Epoch [27/40], Step [600/782] Loss: 1.6125\n",
      "Epoch [27/40], Step [700/782] Loss: 1.7223\n",
      "Epoch [28/40], Step [100/782] Loss: 1.9447\n",
      "Epoch [28/40], Step [200/782] Loss: 1.6265\n",
      "Epoch [28/40], Step [300/782] Loss: 1.6094\n",
      "Epoch [28/40], Step [400/782] Loss: 1.7256\n",
      "Epoch [28/40], Step [500/782] Loss: 1.7515\n",
      "Epoch [28/40], Step [600/782] Loss: 1.6393\n",
      "Epoch [28/40], Step [700/782] Loss: 1.6115\n",
      "Epoch [29/40], Step [100/782] Loss: 1.6624\n",
      "Epoch [29/40], Step [200/782] Loss: 1.4734\n",
      "Epoch [29/40], Step [300/782] Loss: 1.6720\n",
      "Epoch [29/40], Step [400/782] Loss: 1.7595\n",
      "Epoch [29/40], Step [500/782] Loss: 1.7856\n",
      "Epoch [29/40], Step [600/782] Loss: 1.6893\n",
      "Epoch [29/40], Step [700/782] Loss: 1.6409\n",
      "Epoch [30/40], Step [100/782] Loss: 1.6574\n",
      "Epoch [30/40], Step [200/782] Loss: 1.6954\n",
      "Epoch [30/40], Step [300/782] Loss: 1.5892\n",
      "Epoch [30/40], Step [400/782] Loss: 1.6715\n",
      "Epoch [30/40], Step [500/782] Loss: 1.7849\n",
      "Epoch [30/40], Step [600/782] Loss: 1.5825\n",
      "Epoch [30/40], Step [700/782] Loss: 1.7368\n",
      "Accuracy of the model on the test images: 38.76 %\n",
      "Epoch [31/40], Step [100/782] Loss: 1.8286\n",
      "Epoch [31/40], Step [200/782] Loss: 1.9816\n",
      "Epoch [31/40], Step [300/782] Loss: 1.7140\n",
      "Epoch [31/40], Step [400/782] Loss: 1.7189\n",
      "Epoch [31/40], Step [500/782] Loss: 1.7950\n",
      "Epoch [31/40], Step [600/782] Loss: 1.7295\n",
      "Epoch [31/40], Step [700/782] Loss: 1.7085\n",
      "Epoch [32/40], Step [100/782] Loss: 1.7045\n",
      "Epoch [32/40], Step [200/782] Loss: 1.5175\n",
      "Epoch [32/40], Step [300/782] Loss: 1.7971\n",
      "Epoch [32/40], Step [400/782] Loss: 1.7951\n",
      "Epoch [32/40], Step [500/782] Loss: 1.6529\n",
      "Epoch [32/40], Step [600/782] Loss: 1.8389\n",
      "Epoch [32/40], Step [700/782] Loss: 1.6141\n",
      "Epoch [33/40], Step [100/782] Loss: 1.6314\n",
      "Epoch [33/40], Step [200/782] Loss: 1.7599\n",
      "Epoch [33/40], Step [300/782] Loss: 1.5112\n",
      "Epoch [33/40], Step [400/782] Loss: 1.7951\n",
      "Epoch [33/40], Step [500/782] Loss: 1.6062\n",
      "Epoch [33/40], Step [600/782] Loss: 1.6040\n",
      "Epoch [33/40], Step [700/782] Loss: 1.7685\n",
      "Epoch [34/40], Step [100/782] Loss: 1.6272\n",
      "Epoch [34/40], Step [200/782] Loss: 1.6087\n",
      "Epoch [34/40], Step [300/782] Loss: 1.5645\n",
      "Epoch [34/40], Step [400/782] Loss: 1.7417\n",
      "Epoch [34/40], Step [500/782] Loss: 1.6869\n",
      "Epoch [34/40], Step [600/782] Loss: 1.6909\n",
      "Epoch [34/40], Step [700/782] Loss: 1.6934\n",
      "Epoch [35/40], Step [100/782] Loss: 1.6416\n",
      "Epoch [35/40], Step [200/782] Loss: 1.8608\n",
      "Epoch [35/40], Step [300/782] Loss: 1.4084\n",
      "Epoch [35/40], Step [400/782] Loss: 1.7418\n",
      "Epoch [35/40], Step [500/782] Loss: 1.6024\n",
      "Epoch [35/40], Step [600/782] Loss: 1.7539\n",
      "Epoch [35/40], Step [700/782] Loss: 1.4566\n",
      "Accuracy of the model on the test images: 38.43 %\n",
      "Epoch [36/40], Step [100/782] Loss: 1.6490\n",
      "Epoch [36/40], Step [200/782] Loss: 1.6356\n",
      "Epoch [36/40], Step [300/782] Loss: 1.9454\n",
      "Epoch [36/40], Step [400/782] Loss: 1.8440\n",
      "Epoch [36/40], Step [500/782] Loss: 1.7347\n",
      "Epoch [36/40], Step [600/782] Loss: 1.6874\n",
      "Epoch [36/40], Step [700/782] Loss: 2.0812\n",
      "Epoch [37/40], Step [100/782] Loss: 1.8416\n",
      "Epoch [37/40], Step [200/782] Loss: 1.5989\n",
      "Epoch [37/40], Step [300/782] Loss: 1.6894\n",
      "Epoch [37/40], Step [400/782] Loss: 1.6085\n",
      "Epoch [37/40], Step [500/782] Loss: 1.6646\n",
      "Epoch [37/40], Step [600/782] Loss: 1.7045\n",
      "Epoch [37/40], Step [700/782] Loss: 1.6067\n",
      "Epoch [38/40], Step [100/782] Loss: 1.9631\n",
      "Epoch [38/40], Step [200/782] Loss: 1.7098\n",
      "Epoch [38/40], Step [300/782] Loss: 1.6854\n",
      "Epoch [38/40], Step [400/782] Loss: 1.7565\n",
      "Epoch [38/40], Step [500/782] Loss: 1.6911\n",
      "Epoch [38/40], Step [600/782] Loss: 1.5909\n",
      "Epoch [38/40], Step [700/782] Loss: 1.6401\n",
      "Epoch [39/40], Step [100/782] Loss: 1.7515\n",
      "Epoch [39/40], Step [200/782] Loss: 1.6118\n",
      "Epoch [39/40], Step [300/782] Loss: 1.5345\n",
      "Epoch [39/40], Step [400/782] Loss: 1.7425\n",
      "Epoch [39/40], Step [500/782] Loss: 1.6072\n",
      "Epoch [39/40], Step [600/782] Loss: 1.7332\n",
      "Epoch [39/40], Step [700/782] Loss: 1.8102\n",
      "Epoch [40/40], Step [100/782] Loss: 1.7814\n",
      "Epoch [40/40], Step [200/782] Loss: 1.6037\n",
      "Epoch [40/40], Step [300/782] Loss: 1.6976\n",
      "Epoch [40/40], Step [400/782] Loss: 1.6430\n",
      "Epoch [40/40], Step [500/782] Loss: 1.7068\n",
      "Epoch [40/40], Step [600/782] Loss: 1.7791\n",
      "Epoch [40/40], Step [700/782] Loss: 1.7014\n",
      "Accuracy of the model on the test images: 38.7 %\n"
     ]
    }
   ],
   "source": [
    "# Train the model\n",
    "total_step = len(train_loader)\n",
    "curr_lr = 0.01\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "\n",
    "        # Forward pass\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "\n",
    "        # Backward and optimize\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        if (i+1) % 100 == 0:\n",
    "            print (\"Epoch [{}/{}], Step [{}/{}] Loss: {:.4f}\"\n",
    "                   .format(epoch+1, num_epochs, i+1, total_step, loss.item()))\n",
    "    if((epoch+1)%5==0):\n",
    "            evaluate(model)\n",
    "    \n",
    "    # Decay learning rate\n",
    "    if (epoch+1) % 6 == 0:\n",
    "        curr_lr /= 4\n",
    "        update_lr(optimizer, curr_lr)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}