part5_torch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import matplotlib.pyplot as plt \n",
    "import torch.nn.functional as F\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32033 15 2\n",
      "torch.Size([182625, 8]) torch.Size([182625])\n",
      "torch.Size([22655, 8]) torch.Size([22655])\n",
      "torch.Size([22866, 8]) torch.Size([22866])\n"
     ]
    }
   ],
   "source": [
    "with open(\"./names.txt\") as f:\n",
    "    words = f.read().splitlines()\n",
    "    print(len(words), max(len(w) for w in words), min(len(w) for w in words))\n",
    "\n",
    "chars = sorted(list(set((''.join(words)))))  #get unique characters \n",
    "stoi = {s:i+1 for i, s in enumerate(chars)}  # map char to int\n",
    "stoi[\".\"] = 0 \n",
    "itos = {i:s for s,i in stoi.items()}         # map int to char\n",
    "vocab_size = len(itos)\n",
    "\n",
    "\n",
    "# build the dataset\n",
    "block_size = 8 # context length: how many characters do we take to predict the next one?\n",
    "\n",
    "def build_dataset(words):  \n",
    "  X, Y = [], []\n",
    "  for w in words:\n",
    "    context = [0] * block_size\n",
    "    for ch in w + '.':\n",
    "      ix = stoi[ch]\n",
    "      X.append(context)\n",
    "      Y.append(ix)\n",
    "      context = context[1:] + [ix] # crop and append\n",
    "\n",
    "  X = torch.tensor(X)\n",
    "  Y = torch.tensor(Y)\n",
    "  print(X.shape, Y.shape)\n",
    "  return X, Y\n",
    "\n",
    "import random\n",
    "random.seed(42)\n",
    "random.shuffle(words)\n",
    "n1 = int(0.8*len(words))\n",
    "n2 = int(0.9*len(words))\n",
    "\n",
    "Xtr, Ytr = build_dataset(words[:n1])\n",
    "Xdev, Ydev = build_dataset(words[n1:n2])\n",
    "Xte, Yte = build_dataset(words[n2:])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_dim = 8 \n",
    "n_hidden = 164\n",
    "class WaveNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.emb = nn.Embedding(vocab_size,n_dim)\n",
    "        self.seq = nn.Sequential(\n",
    "            \n",
    "            nn.BatchNorm1d(n_hidden),\n",
    "            nn.Linear(n_hidden, vocab_size),\n",
    "        nn.BatchNorm1d(vocab_size)\n",
    "        )\n",
    "    def forward(self, x):\n",
    "        emb = self.emb(x).view(-1, n_dim*block_size)\n",
    "        return self.seq(emb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FlattenConsecutive:\n",
    "  \n",
    "  def __init__(self, n):\n",
    "    self.n = n\n",
    "    \n",
    "  def __call__(self, x):\n",
    "    B, T, C = x.shape\n",
    "    x = x.view(B, T//self.n, C*self.n)\n",
    "    if x.shape[1] == 1:\n",
    "      x = x.squeeze(1)\n",
    "    self.out = x\n",
    "    return self.out\n",
    "  \n",
    "  def parameters(self):\n",
    "    return []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = nn.Conv1d(16, 33, 3, stride=4)\n",
    "input = torch.randn(20, 16, 50)\n",
    "output = m(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([20, 33, 12])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "16*0.75"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = torch.randn(1, 16, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "__init__() missing 2 required positional arguments: 'out_channels' and 'kernel_size'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[11], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m nn\u001b[39m.\u001b[39;49mConv1d(a)\n",
      "\u001b[0;31mTypeError\u001b[0m: __init__() missing 2 required positional arguments: 'out_channels' and 'kernel_size'"
     ]
    }
   ],
   "source": [
    "nn.Conv1d(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([20, 33, 24])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = nn.Conv1d(16, 33, 3, stride=2)\n",
    "input = torch.randn(20, 16, 50)\n",
    "output = m(input)\n",
    "output.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = torch.ones(10, 4,5)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}