train_frames_bin_lin.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/stevenabreu/miniconda3/envs/dvsflow/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import tonic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_folder = '../data/bin_1ms_comp/'\n",
    "chars = list(\"AB\")\n",
    "fidxs = list(range(1, 5))\n",
    "paths = sorted([f'{data_folder}/{c}{fi}.npy' for c in chars for fi in fidxs])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## checks"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "check potential dataset class imbalance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A1: 61.1k ->  7.4k (12.1%), A2: 62.6k ->  4.7k ( 7.5%), A3: 62.7k -> 22.1k (35.2%), A4: 70.4k -> 15.5k (22.0%), \n",
      "B1: 63.1k ->  6.9k (10.9%), B2: 62.8k -> 12.2k (19.4%), B3: 65.8k -> 20.1k (30.5%), B4: 62.3k ->  6.4k (10.2%), \n",
      "A: 256.9k ->  49.7k (19.3%)\n",
      "B: 254.0k ->  45.5k (17.9%)\n"
     ]
    }
   ],
   "source": [
    "per_particle = {ch: (0, 0) for ch in chars}\n",
    "for ch in chars:\n",
    "    for fi in fidxs:\n",
    "        path = f'{data_folder}/{ch}{fi}.npy'\n",
    "        data = np.load(path, allow_pickle=True)\n",
    "        over1k = [e for e in data if e.shape[0] >= 1000]\n",
    "        per_particle[ch] = (per_particle[ch][0] + len(over1k), per_particle[ch][1] + data.shape[0])\n",
    "        print(f'{ch}{fi}: {data.shape[0]/1000:4.1f}k -> {len(over1k)/1000:4.1f}k '\n",
    "              f'({len(over1k)/data.shape[0]:5.1%})', end=', ')\n",
    "    print()\n",
    "for ch, (n_over, n_total) in per_particle.items():\n",
    "    print(f'{ch}: {n_total/1000:5.1f}k -> {n_over/1000:5.1f}k ({n_over/n_total:5.1%})')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- no strong imbalance between particles\n",
    "- trials 2 and 4 are imbalanced between particles \n",
    "    - can expect worse generalization over data folds -> balance when training? *not needed, see results below*"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "label = []\n",
    "trial = []\n",
    "for class_idx, ch in enumerate(chars):\n",
    "    for fi in fidxs:\n",
    "        path = f'{data_folder}/{ch}{fi}.npy'\n",
    "        d = np.load(path, allow_pickle=True)\n",
    "        d = d[[e.shape[0] >= 1_000 for e in d]]\n",
    "        data.append(d)\n",
    "        label.append([class_idx] * len(d))\n",
    "        trial.append([fi] * len(d))\n",
    "data = np.concatenate(data)\n",
    "label = np.concatenate(label)\n",
    "trial = np.concatenate(trial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "transform = tonic.transforms.ToImage(sensor_size=(32, 24, 2,))\n",
    "data = np.array([transform(img) for img in data])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(95154, 2, 24, 32)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## train linear model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(95154, 1536)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.reshape(data.shape[0], -1) # 2, 24, 32 -> 1536\n",
    "data.shape"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generalization to new trial\n",
    "\n",
    "Train on three trials, test on the fourth one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing on trial 1, training on all others\n",
      "Train: 80.9k, Test: 14.2k\n",
      "Train A|B: 52.26% | 47.74%\n",
      "Test A|B:  51.77% | 48.23%\n",
      "Train accuracy: 99.00%\n",
      "Test accuracy: 94.78%\n",
      "\n",
      "Testing on trial 2, training on all others\n",
      "Train: 78.3k, Test: 16.9k\n",
      "Train A|B: 57.45% | 42.55%\n",
      "Test A|B:  27.83% | 72.17%\n",
      "Train accuracy: 98.96%\n",
      "Test accuracy: 96.11%\n",
      "\n",
      "Testing on trial 3, training on all others\n",
      "Train: 53.0k, Test: 42.2k\n",
      "Train A|B: 52.03% | 47.97%\n",
      "Test A|B:  52.39% | 47.61%\n",
      "Train accuracy: 98.50%\n",
      "Test accuracy: 96.98%\n",
      "\n",
      "Testing on trial 4, training on all others\n",
      "Train: 73.3k, Test: 21.9k\n",
      "Train A|B: 46.61% | 53.39%\n",
      "Test A|B:  70.89% | 29.11%\n",
      "Train accuracy: 98.98%\n",
      "Test accuracy: 96.33%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# train linear classifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "tracc = []\n",
    "teacc = []\n",
    "\n",
    "for test_trial in range(1, 5):\n",
    "    # train test split\n",
    "    train_idxs = trial != test_trial\n",
    "    test_idxs = trial == test_trial\n",
    "    train_data = data[train_idxs]\n",
    "    train_labels = label[train_idxs]\n",
    "    train_trials = trial[train_idxs]\n",
    "    test_data = data[test_idxs]\n",
    "    test_labels = label[test_idxs]\n",
    "    test_trials = trial[test_idxs]\n",
    "\n",
    "    # log\n",
    "    print(f'Testing on trial {test_trial}, training on all others')\n",
    "    print(f'  Train: {train_data.shape[0]/1000:3.0f}k, Test: {test_data.shape[0]/1000:3.0f}k')\n",
    "    ntot = np.sum(train_labels == 0) + np.sum(train_labels == 1)\n",
    "    print('  Class balance: ')\n",
    "    print(f'train {np.sum(train_labels == 0)/ntot:.0%} | {np.sum(train_labels == 1)/ntot:.0%}', end=', ')\n",
    "    ntot = np.sum(test_labels == 0) + np.sum(test_labels == 1)\n",
    "    print(f'test  {np.sum(test_labels == 0)/ntot:.0%} | {np.sum(test_labels == 1)/ntot:.0%}')\n",
    "\n",
    "    # train pipeline: standard scaler + logistic regression\n",
    "    pipeline = Pipeline([\n",
    "        ('scaler', StandardScaler()), \n",
    "        ('model', LogisticRegression(max_iter=1000))\n",
    "    ])\n",
    "    pipeline.fit(train_data, train_labels)\n",
    "    tracc.append(pipeline.score(train_data, train_labels))\n",
    "    teacc.append(pipeline.score(test_data, test_labels))\n",
    "    print(f'Train accuracy: {tracc[-1]:.2%}')\n",
    "    print(f'Test accuracy:  {teacc[-1]:.2%}')\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: 98.86% +- 0.21%\n",
      "testing:  96.05% +- 0.80%\n"
     ]
    }
   ],
   "source": [
    "print(f'training: {tracc.mean()/100:.2%} +- {tracc.std()/100:.2%}')\n",
    "print(f'testing:  {teacc.mean()/100:.2%} +- {teacc.std()/100:.2%}')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generalization within trials\n",
    "\n",
    "train on the first 80% of each trial, test on the remaining 20% of each trial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "transform = tonic.transforms.ToImage(sensor_size=(32, 24, 2,))\n",
    "train_data = []\n",
    "test_data = []\n",
    "train_labels = []\n",
    "test_labels = []\n",
    "train_trials = []\n",
    "test_trials = []\n",
    "for class_idx, ch in enumerate(chars):\n",
    "    for fi in fidxs:\n",
    "        path = f'{data_folder}/{ch}{fi}.npy'\n",
    "        d = np.load(path, allow_pickle=True)\n",
    "        # filter (>1k events), transform to image, flatten\n",
    "        d = d[[e.shape[0] >= 1_000 for e in d]]\n",
    "        d = np.array([transform(img) for img in d]).reshape(-1, 1536)\n",
    "\n",
    "        # random 80/20 split\n",
    "        train_len = d.shape[0] * 4 // 5\n",
    "        test_len = d.shape[0] - train_len\n",
    "        shuffled_idxs = np.random.permutation(d.shape[0])\n",
    "        train_data.append(d[shuffled_idxs[:train_len]])\n",
    "        test_data.append(d[shuffled_idxs[train_len:]])\n",
    "\n",
    "        train_labels += [class_idx] * train_len\n",
    "        test_labels += [class_idx] * test_len\n",
    "        train_trials += [fi] * train_len\n",
    "        test_trials += [fi] * test_len\n",
    "train_data = np.concatenate(train_data)\n",
    "test_data = np.concatenate(test_data)\n",
    "train_labels = np.array(train_labels)\n",
    "test_labels = np.array(test_labels)\n",
    "train_trials = np.array(train_trials)\n",
    "test_trials = np.array(test_trials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train accuracy: 98.75%\n",
      "Test accuracy: 97.01% (0.52:0.48), 96.95% (0.28:0.72), 98.72% (0.52:0.48), 97.76% (0.71:0.29), \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# train linear classifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# train pipeline: standard scaler + logistic regression\n",
    "pipeline = Pipeline([\n",
    "    ('scaler', StandardScaler()), \n",
    "    ('model', LogisticRegression(max_iter=1000))\n",
    "])\n",
    "pipeline.fit(train_data, train_labels)\n",
    "\n",
    "tracc = pipeline.score(train_data, train_labels)\n",
    "print(f'Train accuracy: {tracc:.2%}')\n",
    "\n",
    "print(f'Test accuracy: ', end='')\n",
    "teacc = {}\n",
    "for te_idx in range(1, 5):\n",
    "    te_mask = test_trials == te_idx\n",
    "    teacc[te_idx] = pipeline.score(test_data[te_mask], test_labels[te_mask])\n",
    "    print(f'{teacc[te_idx]:.2%}', end=' ')\n",
    "    tr_mask = train_trials == te_idx\n",
    "    ntot = np.sum(train_labels[tr_mask] == 0) + np.sum(train_labels[tr_mask] == 1)\n",
    "    print(f'({np.sum(train_labels[tr_mask] == 0)/ntot:.2f}:{np.sum(train_labels[tr_mask] == 1)/ntot:.2f})', end=', ')\n",
    "print('\\n')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Average accuracies:\n",
    "- Lin: 97.61% test 98.75% train\n",
    "- FNN: 97.51% test 99.52% train\n",
    "- CNN: 97.09% test 99.67% train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dvsflow",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "c205903518a6b2b927394c4307ce9e25709035f7bdde4eb5082c9964fe6e5041"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}