nn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "7427b23f-5784-4d73-97c7-7234d561b855",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make simple pytorch neural network\n",
    "import torch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from torch.optim import Adam\n",
    "#import resample\n",
    "from sklearn.utils import resample\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from tqdm import tqdm, trange\n",
    "import numpy as np\n",
    "import torchmetrics \n",
    "import enlighten\n",
    "import pickle\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "761adeda-12aa-4ac2-9901-3e625c7323da",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load data\n",
    "data = np.load('data/processed/concatenated_features.npy')\n",
    "\n",
    "SEED = 1706\n",
    "LEARNING_RATE = 0.001\n",
    "EPOCHS = 50\n",
    "BATCH_SIZE = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "06096dd1-4ae3-4696-8145-8f223326f123",
   "metadata": {},
   "outputs": [],
   "source": [
    "# split into training and test set\n",
    "x_train, x_test, y_train, y_test = train_test_split(data[:, :-1], data[:, -1], test_size=0.2, random_state=SEED, shuffle=True)\n",
    "\n",
    "# upsample minority class\n",
    "x_minority = x_train[y_train == 0]\n",
    "y_minority = y_train[y_train == 0]\n",
    "x_majority = x_train[y_train == 1]\n",
    "y_majority = y_train[y_train == 1]\n",
    "\n",
    "x_majority_subsample, y_majority_subsample = resample(x_majority, y_majority, replace=False, n_samples=x_minority.shape[0], random_state=SEED)\n",
    "\n",
    "x_train_balanced = np.concatenate((x_majority_subsample, x_minority))\n",
    "y_train_balanced = np.concatenate((y_majority_subsample, y_minority))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "e27a40a6-8ad5-4684-86f8-ff8b03206bcb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.10.1+cu113\n",
      "Using cuda device\n",
      "Fri Dec 24 01:52:50 2021       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 470.82.01    Driver Version: 470.82.01    CUDA Version: 11.4     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  NVIDIA A100-SXM...  On   | 00000000:07:00.0 Off |                    0 |\n",
      "| N/A   27C    P0    56W / 400W |   3760MiB / 40536MiB |      0%      Default |\n",
      "|                               |                      |             Disabled |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   1  NVIDIA A100-SXM...  On   | 00000000:0F:00.0 Off |                    0 |\n",
      "| N/A   20C    P0    50W / 400W |      3MiB / 40536MiB |      0%      Default |\n",
      "|                               |                      |             Disabled |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   2  NVIDIA A100-SXM...  On   | 00000000:47:00.0 Off |                    0 |\n",
      "| N/A   20C    P0    49W / 400W |      3MiB / 40536MiB |      0%      Default |\n",
      "|                               |                      |             Disabled |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   3  NVIDIA A100-SXM...  On   | 00000000:4E:00.0 Off |                    0 |\n",
      "| N/A   21C    P0    51W / 400W |      3MiB / 40536MiB |      0%      Default |\n",
      "|                               |                      |             Disabled |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   4  NVIDIA A100-SXM...  On   | 00000000:87:00.0 Off |                    0 |\n",
      "| N/A   27C    P0    52W / 400W |      3MiB / 40536MiB |      0%      Default |\n",
      "|                               |                      |             Disabled |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   5  NVIDIA A100-SXM...  On   | 00000000:90:00.0 Off |                    0 |\n",
      "| N/A   23C    P0    51W / 400W |      3MiB / 40536MiB |      0%      Default |\n",
      "|                               |                      |             Disabled |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   6  NVIDIA A100-SXM...  On   | 00000000:B7:00.0 Off |                    0 |\n",
      "| N/A   24C    P0    49W / 400W |      3MiB / 40536MiB |      0%      Default |\n",
      "|                               |                      |             Disabled |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   7  NVIDIA A100-SXM...  On   | 00000000:BD:00.0 Off |                    0 |\n",
      "| N/A   24C    P0    49W / 400W |      3MiB / 40536MiB |      0%      Default |\n",
      "|                               |                      |             Disabled |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|    0   N/A  N/A    694722      C   ...vs/pedestrians/bin/python     3757MiB |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "print(torch.__version__)\n",
    "\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "print(f'Using {device} device')\n",
    "\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    !nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "358a3bb3-fd59-4145-8514-2e9ffa89bf33",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TrainData(Dataset):\n",
    "    \n",
    "    def __init__(self, X_data, y_data):\n",
    "        self.X_data = X_data\n",
    "        self.y_data = y_data\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        return self.X_data[index], self.y_data[index]\n",
    "        \n",
    "    def __len__ (self):\n",
    "        return len(self.X_data)\n",
    "\n",
    "\n",
    "train_data = TrainData(torch.FloatTensor(x_train_balanced), \n",
    "                       torch.FloatTensor(y_train_balanced))\n",
    "## test data    \n",
    "class TestData(Dataset):\n",
    "    \n",
    "    def __init__(self, X_data):\n",
    "        self.X_data = X_data\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        return self.X_data[index]\n",
    "        \n",
    "    def __len__ (self):\n",
    "        return len(self.X_data)\n",
    "    \n",
    "\n",
    "test_data = TestData(torch.FloatTensor(x_test))\n",
    "\n",
    "train_dataloader = DataLoader(train_data, batch_size=BATCH_SIZE, shuffle=True)\n",
    "test_dataloader = DataLoader(test_data, batch_size=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "8a39b3c1-7a33-470a-926b-fc939a34603d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class NeuralNetwork(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(NeuralNetwork, self).__init__()\n",
    "        \n",
    "        self.linear_1 = nn.Linear(5595, 10024)\n",
    "        self.linear_2 = nn.Linear(10024, 1024)\n",
    "        self.linear_3 = nn.Linear(1024, 256)\n",
    "        self.output_layer = nn.Linear(256, 1)\n",
    "        \n",
    "        self.relu = nn.ReLU()\n",
    "        self.dropout = nn.Dropout(p=0.3)\n",
    "        self.batch_norm1 = nn.BatchNorm1d(10024)\n",
    "        self.batch_norm2 = nn.BatchNorm1d(1024)\n",
    "        self.batch_norm3 = nn.BatchNorm1d(256)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.linear_1(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.batch_norm1(x)\n",
    "        x = self.dropout(x)\n",
    "        x = self.linear_2(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.batch_norm2(x)\n",
    "        x = self.dropout(x)\n",
    "        x = self.linear_3(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.batch_norm3(x)\n",
    "        x = self.dropout(x)\n",
    "        x = self.output_layer(x)\n",
    "        \n",
    "        return x\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "67a575cf-9101-4df8-899f-3b95e4ccc298",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NeuralNetwork(\n",
      "  (linear_1): Linear(in_features=5595, out_features=10024, bias=True)\n",
      "  (linear_2): Linear(in_features=10024, out_features=1024, bias=True)\n",
      "  (linear_3): Linear(in_features=1024, out_features=256, bias=True)\n",
      "  (output_layer): Linear(in_features=256, out_features=1, bias=True)\n",
      "  (relu): ReLU()\n",
      "  (dropout): Dropout(p=0.3, inplace=False)\n",
      "  (batch_norm1): BatchNorm1d(10024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  (batch_norm2): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  (batch_norm3): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "model = NeuralNetwork()\n",
    "model.to(device)\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "2b87112f-166d-407e-8107-d10a1f3b201a",
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.BCEWithLogitsLoss()\n",
    "optimizer = Adam(model.parameters(), lr=LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "7778b2e8-6b53-49c6-93cf-d2414b4e2617",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "a4976ae9-1b2d-4c25-971f-cb8581520196",
   "metadata": {},
   "outputs": [],
   "source": [
    "def binary_acc(y_pred, y_test):\n",
    "    y_pred_tag = torch.round(torch.sigmoid(y_pred))\n",
    "\n",
    "    correct_results_sum = (y_pred_tag == y_test).sum().float()\n",
    "    acc = correct_results_sum/y_test.shape[0]\n",
    "    acc = torch.round(acc * 100)\n",
    "    \n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "ccd63a55-1b53-43e1-8ed2-8708038872cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>\n",
       "</style>\n",
       "<div class=\"enlighten\">\n",
       "  <div class=\"enlighten-bar\">\n",
       "    <pre>Epochs 100%|████████████████████████████████████████████████████| 50/50 [02:17&lt;00:00, 0.37 epochs/s]</pre>\n",
       "  </div>\n",
       "  <div class=\"enlighten-bar\">\n",
       "    <pre>batch 9694 batches [02:17, 70.90 batches/s]                                                         </pre>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 001: | Loss: 0.02862 | Acc: 98.989 | Acc_2: 98.851\n",
      "Epoch 002: | Loss: 0.02195 | Acc: 99.086 | Acc_2: 99.098\n",
      "Epoch 003: | Loss: 0.03080 | Acc: 98.986 | Acc_2: 98.644\n",
      "Epoch 004: | Loss: 0.03707 | Acc: 98.896 | Acc_2: 98.577\n",
      "Epoch 005: | Loss: 0.02809 | Acc: 98.911 | Acc_2: 98.943\n",
      "Epoch 006: | Loss: 0.03273 | Acc: 98.915 | Acc_2: 98.686\n",
      "Epoch 007: | Loss: 0.04067 | Acc: 98.876 | Acc_2: 98.443\n",
      "Epoch 008: | Loss: 0.03319 | Acc: 98.872 | Acc_2: 98.732\n",
      "Epoch 009: | Loss: 0.03063 | Acc: 98.866 | Acc_2: 98.722\n",
      "Epoch 010: | Loss: 0.02228 | Acc: 98.903 | Acc_2: 99.108\n",
      "Epoch 011: | Loss: 0.02074 | Acc: 98.938 | Acc_2: 99.201\n",
      "Epoch 012: | Loss: 0.02542 | Acc: 98.960 | Acc_2: 99.026\n",
      "Epoch 013: | Loss: 0.02665 | Acc: 98.965 | Acc_2: 98.985\n",
      "Epoch 014: | Loss: 0.02956 | Acc: 98.980 | Acc_2: 98.974\n",
      "Epoch 015: | Loss: 0.02643 | Acc: 98.985 | Acc_2: 98.974\n",
      "Epoch 016: | Loss: 0.02895 | Acc: 98.985 | Acc_2: 98.881\n",
      "Epoch 017: | Loss: 0.02393 | Acc: 98.998 | Acc_2: 99.000\n",
      "Epoch 018: | Loss: 0.02799 | Acc: 98.993 | Acc_2: 98.789\n",
      "Epoch 019: | Loss: 0.02732 | Acc: 98.997 | Acc_2: 98.938\n",
      "Epoch 020: | Loss: 0.02229 | Acc: 99.012 | Acc_2: 99.139\n",
      "Epoch 021: | Loss: 0.03462 | Acc: 99.007 | Acc_2: 98.582\n",
      "Epoch 022: | Loss: 0.02493 | Acc: 99.008 | Acc_2: 98.938\n",
      "Epoch 023: | Loss: 0.02379 | Acc: 99.014 | Acc_2: 99.108\n",
      "Epoch 024: | Loss: 0.02217 | Acc: 99.027 | Acc_2: 99.170\n",
      "Epoch 025: | Loss: 0.02376 | Acc: 99.033 | Acc_2: 98.990\n",
      "Epoch 026: | Loss: 0.01878 | Acc: 99.045 | Acc_2: 99.284\n",
      "Epoch 027: | Loss: 0.01997 | Acc: 99.057 | Acc_2: 99.253\n",
      "Epoch 028: | Loss: 0.01854 | Acc: 99.069 | Acc_2: 99.268\n",
      "Epoch 029: | Loss: 0.02248 | Acc: 99.077 | Acc_2: 99.206\n",
      "Epoch 030: | Loss: 0.03766 | Acc: 99.063 | Acc_2: 98.469\n",
      "Epoch 031: | Loss: 0.02874 | Acc: 99.060 | Acc_2: 98.758\n",
      "Epoch 032: | Loss: 0.02427 | Acc: 99.063 | Acc_2: 98.990\n",
      "Epoch 033: | Loss: 0.02473 | Acc: 99.061 | Acc_2: 98.902\n",
      "Epoch 034: | Loss: 0.02187 | Acc: 99.068 | Acc_2: 99.072\n",
      "Epoch 035: | Loss: 0.02723 | Acc: 99.064 | Acc_2: 98.887\n",
      "Epoch 036: | Loss: 0.02510 | Acc: 99.069 | Acc_2: 99.041\n",
      "Epoch 037: | Loss: 0.02000 | Acc: 99.074 | Acc_2: 99.222\n",
      "Epoch 038: | Loss: 0.02109 | Acc: 99.081 | Acc_2: 99.201\n",
      "Epoch 039: | Loss: 0.03001 | Acc: 99.080 | Acc_2: 98.954\n",
      "Epoch 040: | Loss: 0.02775 | Acc: 99.080 | Acc_2: 98.881\n",
      "Epoch 041: | Loss: 0.01881 | Acc: 99.086 | Acc_2: 99.191\n",
      "Epoch 042: | Loss: 0.02040 | Acc: 99.092 | Acc_2: 99.273\n",
      "Epoch 043: | Loss: 0.01817 | Acc: 99.098 | Acc_2: 99.284\n",
      "Epoch 044: | Loss: 0.02236 | Acc: 99.100 | Acc_2: 99.015\n",
      "Epoch 045: | Loss: 0.02209 | Acc: 99.104 | Acc_2: 99.175\n",
      "Epoch 046: | Loss: 0.02204 | Acc: 99.107 | Acc_2: 99.093\n",
      "Epoch 047: | Loss: 0.01979 | Acc: 99.111 | Acc_2: 99.191\n",
      "Epoch 048: | Loss: 0.01949 | Acc: 99.116 | Acc_2: 99.263\n",
      "Epoch 049: | Loss: 0.02067 | Acc: 99.119 | Acc_2: 99.134\n",
      "Epoch 050: | Loss: 0.02098 | Acc: 99.121 | Acc_2: 99.191\n",
      "tensor(0.9912, device='cuda:0') tensor(0.9912, device='cuda:0') (tensor([0.5577, 0.5577, 0.5577,  ..., 1.0000, 1.0000, 1.0000], device='cuda:0'), tensor([1.0000e+00, 1.0000e+00, 1.0000e+00,  ..., 6.4704e-06, 3.2352e-06,\n",
      "        0.0000e+00], device='cuda:0'), tensor([-18.6682, -18.6681, -18.6681,  ...,  50.5627,  52.0135,  58.6947],\n",
      "       device='cuda:0')) tensor(0.9996, device='cuda:0') tensor(0.9938, device='cuda:0') tensor(0.9886, device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "manager = enlighten.get_manager()\n",
    "epoch_tick = manager.counter(total=EPOCHS, desc='Epochs', unit='epochs')\n",
    "batch_tick = manager.counter(total=int(x_train_balanced.__len__()/BATCH_SIZE), desc='batch', unit='batches')\n",
    "\n",
    "_acc = torchmetrics.Accuracy().to(device)\n",
    "_f1 = torchmetrics.F1().to(device)\n",
    "_prcu = torchmetrics.PrecisionRecallCurve().to(device)\n",
    "_av = torchmetrics.AveragePrecision().to(device)\n",
    "_pre = torchmetrics.Precision().to(device)\n",
    "_recall = torchmetrics.Recall().to(device)\n",
    "\n",
    "model.train()\n",
    "for e in range(1, EPOCHS+1):\n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    for X_batch, y_batch in train_dataloader:\n",
    "\n",
    "        X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        \n",
    "        y_pred = model(X_batch)\n",
    "        \n",
    "        acc2 = binary_acc(y_pred, y_batch.unsqueeze(1))\n",
    "        \n",
    "        loss = criterion(y_pred, y_batch.unsqueeze(1))\n",
    "        acc = _acc(y_pred, y_batch.int().unsqueeze(1))\n",
    "        f1 = _f1(y_pred, y_batch.int().unsqueeze(1))\n",
    "        prcu = _prcu(y_pred, y_batch.int().unsqueeze(1))\n",
    "        av = _av(y_pred, y_batch.int().unsqueeze(1))\n",
    "        pre = _pre(y_pred, y_batch.int().unsqueeze(1))\n",
    "        recall = _recall(y_pred, y_batch.int().unsqueeze(1))\n",
    "        \n",
    "        \n",
    "        val_loss = criterion\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc2.item()\n",
    "        batch_tick.update()\n",
    "    epoch_tick.update()\n",
    "\n",
    "    print(f'Epoch {e+0:03}: | Loss: {epoch_loss/len(train_dataloader):.5f} | Acc: {_acc.compute()*100:.3f} | Acc_2: {epoch_acc/len(train_dataloader):.3f}')\n",
    "    \n",
    "manager.stop()\n",
    "\n",
    "\n",
    "print(_acc.compute(), _f1.compute(), _prcu.compute(), _av.compute(), _pre.compute(), _recall.compute())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "9b4d5c11-9d6b-4bf8-996d-109b159e4fbf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 99.121 | F1: 99.119  | Average Precision: 99.958 | Precision: 99.377 | Recall: 98.862\n"
     ]
    }
   ],
   "source": [
    "print(f\"Accuracy: {_acc.compute()*100:.3f} | F1: {_f1.compute()*100:.3f}  | Average Precision: {_av.compute()*100:.3f} | Precision: {_pre.compute()*100:.3f} | Recall: {_recall.compute()*100:.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "b32b7921-32ff-4961-ab15-525d848f36b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_list = []\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    for X_batch in test_dataloader:\n",
    "        X_batch = X_batch.to(device)\n",
    "        y_test_pred = model(X_batch)\n",
    "        y_test_pred = torch.sigmoid(y_test_pred)\n",
    "        y_pred_tag = torch.round(y_test_pred)\n",
    "        y_pred_list.append(y_pred_tag.cpu().numpy())\n",
    "\n",
    "        \n",
    "        y_pred_list = [a.squeeze() for a in y_pred_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "89853d51-1e6f-478b-a13b-d78e2fccd5ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1510,   30],\n",
       "       [ 455, 7660]])"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, y_pred_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "6f1e4a0e-d56c-450a-9b97-7db5741f2cd1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.77      0.98      0.86      1540\n",
      "         1.0       1.00      0.94      0.97      8115\n",
      "\n",
      "    accuracy                           0.95      9655\n",
      "   macro avg       0.88      0.96      0.92      9655\n",
      "weighted avg       0.96      0.95      0.95      9655\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, y_pred_list))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b4313ba-48cb-422e-b376-970a7e7e6995",
   "metadata": {},
   "source": [
    "## Shuffle a column at a time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "4c2cd2d6-4e07-4256-a3a5-7ea2283a4972",
   "metadata": {},
   "outputs": [],
   "source": [
    "# same_characteristic\n",
    "same_characteristic = []\n",
    "for j in range(373):\n",
    "    same_characteristic.append([i*373+j for i in range(15)])\n",
    "    \n",
    "def iter_list(l):\n",
    "    for i in l:\n",
    "        yield i\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "564eed3b-3c72-4d9b-827c-45cac7de931e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>\n",
       "</style>\n",
       "<div class=\"enlighten\">\n",
       "  <div class=\"enlighten-bar\">\n",
       "    <pre>Shuffle   1%|▌                                           |   4/373 [00:59&lt;1h 29:60, 0.07 Shuffles/s]</pre>\n",
       "  </div>\n",
       "  <div class=\"enlighten-bar\">\n",
       "    <pre>Epoch  67%|███████████████████████████████████▍                 | 10/15 [00:05&lt;00:02, 2.01 Epochs/s]</pre>\n",
       "  </div>\n",
       "  <div class=\"enlighten-bar\">\n",
       "    <pre>Batch 100%|████████████████████████████████████████████████████| 49/49 [00:00&lt;00:00, 102.27 Batch/s]</pre>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 015: | Loss: 0.11266 | Acc: 95.531\n",
      "{'id_0': [{'Accuracy': 0.9386846423149109, 'F1': 0.9627673029899597, 'Precision': 0.9831727743148804, 'Recall': 0.943191647529602, 'Precision-Recall Curve': (tensor([0.8405, 0.9832, 1.0000]), tensor([1.0000, 0.9432, 0.0000]), tensor([0., 1.]))}]}\n",
      "Epoch 015: | Loss: 0.11192 | Acc: 95.531\n",
      "{'id_1': [{'Accuracy': 0.9336095452308655, 'F1': 0.9592421054840088, 'Precision': 0.9909353852272034, 'Recall': 0.9295132756233215, 'Precision-Recall Curve': (tensor([0.8405, 0.9909, 1.0000]), tensor([1.0000, 0.9295, 0.0000]), tensor([0., 1.]))}]}\n",
      "Epoch 015: | Loss: 0.10932 | Acc: 95.959\n",
      "{'id_2': [{'Accuracy': 0.9287415742874146, 'F1': 0.9562229514122009, 'Precision': 0.9885541200637817, 'Recall': 0.9259396195411682, 'Precision-Recall Curve': (tensor([0.8405, 0.9886, 1.0000]), tensor([1.0000, 0.9259, 0.0000]), tensor([0., 1.]))}]}\n",
      "Epoch 015: | Loss: 0.12515 | Acc: 94.980\n",
      "{'id_3': [{'Accuracy': 0.9260486960411072, 'F1': 0.954591691493988, 'Precision': 0.9863319993019104, 'Recall': 0.9248305559158325, 'Precision-Recall Curve': (tensor([0.8405, 0.9863, 1.0000]), tensor([1.0000, 0.9248, 0.0000]), tensor([0., 1.]))}]}\n"
     ]
    }
   ],
   "source": [
    "EPOCHS_shuffle = 15\n",
    "BATCH_SIZE = 256\n",
    "\n",
    "manager = enlighten.get_manager()\n",
    "row_tick = manager.counter(total=373, desc='Shuffle', unit='Shuffles')\n",
    "\n",
    "group_dict = {}\n",
    "\n",
    "\n",
    "# copy x_train_balanced\n",
    "   \n",
    "for m, i in enumerate(iter_list(same_characteristic)):\n",
    "    x_train_shuffle = x_train_balanced.copy()\n",
    "    for j in i:\n",
    "        np.random.shuffle(x_train_shuffle[:, i])\n",
    "\n",
    "    for layer in model.children():\n",
    "        if hasattr(layer, 'reset_parameters'):\n",
    "            layer.reset_parameters()\n",
    "    \n",
    "    epoch_tick = manager.counter(total=EPOCHS_shuffle, desc='Epoch', unit='Epochs', leave=False)\n",
    "    \n",
    "    model.train()\n",
    "    for e in range(1, EPOCHS_shuffle+1):\n",
    "        epoch_loss = 0\n",
    "        epoch_acc = 0\n",
    "        batch_tick = manager.counter(total=len(train_dataloader), desc='Batch', unit='Batch', leave=False)\n",
    "        for X_batch, y_batch in train_dataloader:\n",
    "            X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            y_pred = model(X_batch)\n",
    "\n",
    "            loss = criterion(y_pred, y_batch.unsqueeze(1))\n",
    "            acc = binary_acc(y_pred, y_batch.unsqueeze(1))\n",
    "\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "            epoch_acc += acc.item()\n",
    "            batch_tick.update()\n",
    "        batch_tick.close()\n",
    "        epoch_tick.update()\n",
    "    epoch_tick.close()\n",
    "    row_tick.update()\n",
    "    print(f'Epoch {e+0:03}: | Loss: {epoch_loss/len(train_dataloader):.5f} | Acc: {epoch_acc/len(train_dataloader):.3f}')\n",
    "    \n",
    "    y_pred_list = []\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for X_batch in test_dataloader:\n",
    "            X_batch = X_batch.to(device)\n",
    "            y_test_pred = model(X_batch)\n",
    "            y_test_pred = torch.sigmoid(y_test_pred)\n",
    "            y_pred_tag = torch.round(y_test_pred)\n",
    "            y_pred_list.append(y_pred_tag)\n",
    "        \n",
    "        y_pred_list = torch.concat(y_pred_list)\n",
    "            \n",
    "        acc = torchmetrics.functional.accuracy(y_pred_list.cpu(), torch.Tensor(y_test).int().cpu())\n",
    "        f1 = torchmetrics.functional.f1(y_pred_list.cpu(), torch.Tensor(y_test).int().cpu())\n",
    "        pre = torchmetrics.functional.precision(y_pred_list.cpu(), torch.Tensor(y_test).int().cpu())\n",
    "        recall = torchmetrics.functional.recall(y_pred_list.cpu(), torch.Tensor(y_test).int().cpu())\n",
    "        prcu = torchmetrics.functional.precision_recall_curve(y_pred_list.cpu(), torch.Tensor(y_test).int().cpu(), num_classes=1)\n",
    "        \n",
    "        single_dict = {f'id_{m}': [{\"Accuracy\": float(acc), \"F1\": float(f1), \"Precision\": float(pre), \"Recall\": float(recall), \"Precision-Recall Curve\": prcu}]}\n",
    "        print(single_dict)\n",
    "        \n",
    "        group_dict.update(single_dict)\n",
    "        pickle.dump(group_dict, open(f\"models/{m}_shuffle_results.pkl\", \"wb\"))\n",
    "        \n",
    "manager.stop()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "c93a5ed5-f46c-484c-8a38-9c369c4d4e7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 1, 1,  ..., 1, 1, 0], dtype=torch.int32)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.Tensor(y_test).int()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "fdae5a69-a7ad-4b43-b0b2-d00a3fb83eff",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame.from_dict(single_dict[list(single_dict.keys())[0]])\n",
    "df.index =list(single_dict.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "9a0590d0-c76a-418b-ad7e-c04e027e5e9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Precision-Recall Curve</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>id_2</th>\n",
       "      <td>tensor(0.9243)</td>\n",
       "      <td>tensor(0.9534)</td>\n",
       "      <td>tensor(0.9886)</td>\n",
       "      <td>tensor(0.9205)</td>\n",
       "      <td>([tensor(0.8405), tensor(0.9886), tensor(1.)],...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Accuracy              F1       Precision          Recall  \\\n",
       "id_2  tensor(0.9243)  tensor(0.9534)  tensor(0.9886)  tensor(0.9205)   \n",
       "\n",
       "                                 Precision-Recall Curve  \n",
       "id_2  ([tensor(0.8405), tensor(0.9886), tensor(1.)],...  "
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "30ea8c47-4c00-4939-bdb5-69af47ae0dea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "91.0"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25ed88eb-ce26-4c93-b6ea-8f65c66f0031",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pedestrians",
   "language": "python",
   "name": "pedestrians"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}