Kim-2014.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from apex import amp\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch\n",
    "import torch.nn.functional as f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_neg = open('/home/ahmed/Desktop/rt-polaritydata/rt-polaritydata/rt-polarity.neg','r', encoding = \"ISO-8859-1\")\n",
    "file_pos = open('/home/ahmed/Desktop/rt-polaritydata/rt-polaritydata/rt-polarity.pos','r', encoding = \"ISO-8859-1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = []\n",
    "labels = []\n",
    "with open('/home/ahmed/Desktop/rt-polaritydata/rt-polaritydata/rt-polarity.neg','r', encoding = \"ISO-8859-1\") as myfile:\n",
    "      for line in myfile:\n",
    "            dataset.append(line)\n",
    "            labels.append(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/home/ahmed/Desktop/rt-polaritydata/rt-polaritydata/rt-polarity.pos','r', encoding = \"ISO-8859-1\") as myfile:\n",
    "      for line in myfile:\n",
    "            dataset.append(line)\n",
    "            labels.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models.word2vec import Word2Vec\n",
    "from gensim.models import KeyedVectors\n",
    "model = KeyedVectors.load_word2vec_format('/home/ahmed/Desktop/GoogleNews-vectors-negative300.bin.gz', binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "dataset , labels = shuffle(dataset,labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = dataset[:8000]\n",
    "test_dataset = dataset[8000:]\n",
    "train_labels = labels[:8000]\n",
    "test_labels = labels[8000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8000"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch(dataset,labels):\n",
    "    batch_size = 512\n",
    "    num_batches = int(len(dataset) / 512)\n",
    "    for i in range(num_batches+1):\n",
    "        yield(dataset[i*batch_size:min(len(dataset),(i+1)*batch_size)],\n",
    "              labels[i*batch_size:min(len(dataset),(i+1)*batch_size)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_list = model.index2word\n",
    "index_mapping = {}\n",
    "for i,w in enumerate(word_list):\n",
    "    index_mapping[w] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "clean_dataset_train = []\n",
    "for sent in train_dataset:\n",
    "    sent = re.sub(r'[\\n\\r]+',' ',sent)\n",
    "    sent = re.sub(r'[\\'\\\"\\.\\-\\,]+','',sent)\n",
    "    clean_dataset_train.append(sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_dataset_test = []\n",
    "for sent in test_dataset:\n",
    "    sent = re.sub(r'[\\n\\r]+',' ',sent)\n",
    "    sent = re.sub(r'[\\'\\\"\\.\\-\\,]+','',sent)\n",
    "    clean_dataset_test.append(sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_indexed_dataset = []\n",
    "for sent in clean_dataset_train:\n",
    "    encoded_sent = []\n",
    "    sent = sent.split(' ')\n",
    "    for word in sent:\n",
    "        if word in index_mapping:\n",
    "            encoded_sent.append(index_mapping[word])\n",
    "    train_indexed_dataset.append(encoded_sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_indexed_dataset = []\n",
    "for sent in clean_dataset_test:\n",
    "    encoded_sent = []\n",
    "    sent = sent.split(' ')\n",
    "    for word in sent:\n",
    "        if word in index_mapping:\n",
    "            encoded_sent.append(index_mapping[word])\n",
    "    test_indexed_dataset.append(encoded_sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "googleNews_vectors = model.vectors\n",
    "googleNews_vectors = torch.from_numpy(googleNews_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad_vector = torch.rand(1,300)\n",
    "end_vector = torch.rand(1,300)\n",
    "googleNews_vectors = torch.cat((googleNews_vectors,pad_vector,end_vector),0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3000002, 300])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "googleNews_vectors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_padded_dataset = []\n",
    "corresponding_train_labels = []\n",
    "for sent,label in zip(train_indexed_dataset,train_labels):\n",
    "    length = len(sent)\n",
    "    if length <46:\n",
    "        train_padded_dataset.append([0]+sent+[3000000]*(44-length)+[3000001])\n",
    "        corresponding_train_labels.append(label)\n",
    "    elif length == 44:\n",
    "        train_padded_dataset.append([0]+sent+[3000001])\n",
    "        corresponding_train_labels.append(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_padded_dataset = []\n",
    "corresponding_test_labels = []\n",
    "for sent,label in zip(test_indexed_dataset,test_labels):\n",
    "    length = len(sent)\n",
    "    if length <46:\n",
    "        test_padded_dataset.append([0]+sent+[3000000]*(44-length)+[3000001])\n",
    "        corresponding_test_labels.append(label)\n",
    "    elif length == 44:\n",
    "        test_padded_dataset.append([0]+sent+[3000001])\n",
    "        corresponding_test_labels.append(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = torch.tensor(train_padded_dataset)\n",
    "X_test = torch.tensor(test_padded_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([7999, 46])\n",
      "torch.Size([2662, 46])\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_train = torch.tensor(train_labels)\n",
    "Y_test = torch.tensor(test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_train = Y_train.unsqueeze(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_test = Y_test.unsqueeze(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CNN_TC(nn.Module):\n",
    "    def __init__(self,googleNews_vectors,Filters):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding.from_pretrained(googleNews_vectors,freeze=True)\n",
    "        self.embedding.weight.requires_grad = False\n",
    "        self.conv1 = nn.Conv1d(1,1,kernel_size=(Filters[0] * 300),stride=(300))\n",
    "        self.conv2 = nn.Conv1d(1,1,kernel_size=(Filters[1] * 300),stride=(300))\n",
    "        self.conv3 = nn.Conv1d(1,1,kernel_size=(Filters[2] * 300),stride=(300))\n",
    "        \n",
    "        self.max_conv1 = nn.MaxPool1d(kernel_size=(15))\n",
    "        self.max_conv2 = nn.MaxPool1d(kernel_size=(11))\n",
    "        self.max_conv3 = nn.MaxPool1d(kernel_size=(9))\n",
    "        \n",
    "        self.FFN1 = nn.Linear(9,100)\n",
    "        self.FFN2 = nn.Linear(100,2)\n",
    "        \n",
    "    def forward(self,inputs):\n",
    "        X = self.embedding(inputs)\n",
    "        X = X.view(X.shape[0],1,-1).contiguous()\n",
    "\n",
    "        max_feature1 = self.max_conv1(f.relu(self.conv1(X)))\n",
    "        max_feature2 = self.max_conv2(f.relu(self.conv2(X)))\n",
    "        max_feature3 = self.max_conv3(f.relu(self.conv3(X)))\n",
    "        features = torch.cat((max_feature1,max_feature2,max_feature3), -1).squeeze(0)\n",
    "                             \n",
    "        output = self.FFN2(f.relu(self.FFN1(features)))\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4805"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_parameters(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CNN_TC(googleNews_vectors,Filters=[3,4,5]).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from apex.optimizers import FusedAdam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters())\n",
    "criterion = nn.CrossEntropyLoss().cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model, optimizer = amp.initialize(model, optimizer,opt_level='O3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.ca"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(Y_hat,Y):\n",
    "    predicted = np.argmax(Y_hat.cpu().data.numpy(),axis=1)\n",
    "    acc = sum([1 if p==y else 0 for p,y in zip(predicted,Y)]) / len(Y_hat)\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.11it/s]\n",
      "3it [00:00, 25.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 1 with Accuracy :0.5423583984375 Loss :45285.83984375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.23it/s]\n",
      "2it [00:00, 16.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 1 with Accuracy :0.5740672700082509\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.21it/s]\n",
      "3it [00:00, 26.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 2 with Accuracy :0.6465576171875 Loss :44271.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.51it/s]\n",
      "2it [00:00, 17.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 2 with Accuracy :0.6205490666254125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 18.50it/s]\n",
      "3it [00:00, 26.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 3 with Accuracy :0.6947998046875 Loss :42051.56640625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.52it/s]\n",
      "2it [00:00, 17.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 3 with Accuracy :0.6316844575082509\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.41it/s]\n",
      "3it [00:00, 26.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 4 with Accuracy :0.7202392578125 Loss :39022.20703125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.03it/s]\n",
      "2it [00:00, 15.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 4 with Accuracy :0.6423138407590759\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.71it/s]\n",
      "3it [00:00, 26.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 5 with Accuracy :0.738330078125 Loss :36213.37890625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.49it/s]\n",
      "2it [00:00, 17.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 5 with Accuracy :0.6445699257425742\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 18.48it/s]\n",
      "3it [00:00, 25.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 6 with Accuracy :0.756640625 Loss :34100.92578125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.53it/s]\n",
      "2it [00:00, 16.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 6 with Accuracy :0.6553797957920792\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.05it/s]\n",
      "3it [00:00, 25.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 7 with Accuracy :0.769921875 Loss :32478.896484375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.13it/s]\n",
      "2it [00:00, 18.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 7 with Accuracy :0.6602626082920792\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 18.43it/s]\n",
      "3it [00:00, 25.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 8 with Accuracy :0.7823486328125 Loss :31139.521484375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 28.71it/s]\n",
      "2it [00:00, 16.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 8 with Accuracy :0.6611714882425742\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 18.32it/s]\n",
      "3it [00:00, 25.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 9 with Accuracy :0.79296875 Loss :30022.34765625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.47it/s]\n",
      "2it [00:00, 17.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 9 with Accuracy :0.6663798215759076\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 18.43it/s]\n",
      "3it [00:00, 25.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 10 with Accuracy :0.800634765625 Loss :29049.72265625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.36it/s]\n",
      "2it [00:00, 15.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 10 with Accuracy :0.665728779909241\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.04it/s]\n",
      "3it [00:00, 24.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 11 with Accuracy :0.8097900390625 Loss :28267.826171875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 28.32it/s]\n",
      "2it [00:00, 17.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 11 with Accuracy :0.6686810282590759\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.72it/s]\n",
      "3it [00:00, 25.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 12 with Accuracy :0.816650390625 Loss :27492.6171875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.08it/s]\n",
      "2it [00:00, 17.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 12 with Accuracy :0.6680074257425743\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 18.50it/s]\n",
      "3it [00:00, 25.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 13 with Accuracy :0.8217529296875 Loss :26747.720703125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 28.70it/s]\n",
      "2it [00:00, 16.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 13 with Accuracy :0.6693320699257427\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.19it/s]\n",
      "3it [00:00, 25.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 14 with Accuracy :0.82822265625 Loss :26021.224609375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.11it/s]\n",
      "2it [00:00, 17.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 14 with Accuracy :0.6689839882425743\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.85it/s]\n",
      "3it [00:00, 25.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 15 with Accuracy :0.8349365234375 Loss :25437.23046875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 28.97it/s]\n",
      "2it [00:00, 17.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 15 with Accuracy :0.6719362365924093\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 18.53it/s]\n",
      "3it [00:00, 25.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 16 with Accuracy :0.8364501953125 Loss :24851.837890625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.54it/s]\n",
      "2it [00:00, 18.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 16 with Accuracy :0.6693095090759077\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.96it/s]\n",
      "3it [00:00, 26.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 17 with Accuracy :0.8400634765625 Loss :24473.390625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.82it/s]\n",
      "2it [00:00, 17.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 17 with Accuracy :0.6673789449257427\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.76it/s]\n",
      "3it [00:00, 26.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 18 with Accuracy :0.843115234375 Loss :24132.240234375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.74it/s]\n",
      "2it [00:00, 17.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 18 with Accuracy :0.6683555074257427\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.72it/s]\n",
      "6it [00:00, 30.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 19 with Accuracy :0.8496337890625 Loss :23527.74609375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "2it [00:00, 17.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 19 with Accuracy :0.6709822349422443\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.42it/s]\n",
      "3it [00:00, 23.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 20 with Accuracy :0.8548828125 Loss :23019.720703125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 26.28it/s]\n",
      "2it [00:00, 16.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 20 with Accuracy :0.6641011757425742\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.02it/s]\n",
      "3it [00:00, 26.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 21 with Accuracy :0.8537353515625 Loss :22913.873046875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.42it/s]\n",
      "2it [00:00, 16.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 21 with Accuracy :0.6637982157590759\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 16.92it/s]\n",
      "6it [00:00, 30.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 22 with Accuracy :0.861572265625 Loss :22184.7109375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "2it [00:00, 18.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 22 with Accuracy :0.6624735715759076\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 18.29it/s]\n",
      "3it [00:00, 24.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 23 with Accuracy :0.862744140625 Loss :21876.548828125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 28.48it/s]\n",
      "2it [00:00, 17.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 23 with Accuracy :0.6605430074257426\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 18.08it/s]\n",
      "3it [00:00, 25.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 24 with Accuracy :0.8591552734375 Loss :22149.126953125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.13it/s]\n",
      "2it [00:00, 17.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 24 with Accuracy :0.6644943791254125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.95it/s]\n",
      "3it [00:00, 24.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 25 with Accuracy :0.8708984375 Loss :21149.60546875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 27.90it/s]\n",
      "2it [00:00, 16.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 25 with Accuracy :0.661822529909241\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.03it/s]\n",
      "3it [00:00, 25.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 26 with Accuracy :0.8720947265625 Loss :20757.263671875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.14it/s]\n",
      "2it [00:00, 17.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 26 with Accuracy :0.660845967409241\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.40it/s]\n",
      "3it [00:00, 25.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 27 with Accuracy :0.8761474609375 Loss :20388.462890625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.16it/s]\n",
      "2it [00:00, 16.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 27 with Accuracy :0.6585898824257426\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.71it/s]\n",
      "3it [00:00, 25.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 28 with Accuracy :0.8731201171875 Loss :20478.41796875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.40it/s]\n",
      "2it [00:00, 17.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 28 with Accuracy :0.656917156559406\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 18.12it/s]\n",
      "3it [00:00, 26.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 29 with Accuracy :0.879638671875 Loss :20030.412109375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.52it/s]\n",
      "2it [00:00, 17.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 29 with Accuracy :0.6572426773927392\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.81it/s]\n",
      "3it [00:00, 25.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 30 with Accuracy :0.8814208984375 Loss :19549.41796875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.32it/s]\n",
      "2it [00:00, 17.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 30 with Accuracy :0.6592183632425742\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.84it/s]\n",
      "3it [00:00, 25.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 31 with Accuracy :0.8841064453125 Loss :19311.947265625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.23it/s]\n",
      "2it [00:00, 17.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 31 with Accuracy :0.6572426773927392\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 18.11it/s]\n",
      "3it [00:00, 24.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 32 with Accuracy :0.886669921875 Loss :18982.275390625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 27.84it/s]\n",
      "2it [00:00, 15.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 32 with Accuracy :0.6542904290429042\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 18.28it/s]\n",
      "3it [00:00, 25.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 33 with Accuracy :0.8872802734375 Loss :18973.06640625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.42it/s]\n",
      "2it [00:00, 17.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 33 with Accuracy :0.6592409240924092\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 18.65it/s]\n",
      "3it [00:00, 26.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 34 with Accuracy :0.889501953125 Loss :18674.109375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.64it/s]\n",
      "2it [00:00, 16.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 34 with Accuracy :0.654010029909241\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 16.77it/s]\n",
      "3it [00:00, 25.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 35 with Accuracy :0.89052734375 Loss :18450.501953125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.09it/s]\n",
      "2it [00:00, 16.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 35 with Accuracy :0.6546836324257426\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.20it/s]\n",
      "3it [00:00, 25.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 36 with Accuracy :0.8930908203125 Loss :18137.65234375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.27it/s]\n",
      "2it [00:00, 17.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 36 with Accuracy :0.6543581115924092\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.30it/s]\n",
      "3it [00:00, 25.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 37 with Accuracy :0.8958984375 Loss :17883.470703125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 28.11it/s]\n",
      "2it [00:00, 16.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 37 with Accuracy :0.6550317141089109\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.29it/s]\n",
      "3it [00:00, 26.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 38 with Accuracy :0.900244140625 Loss :17620.013671875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.19it/s]\n",
      "2it [00:00, 16.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 38 with Accuracy :0.6582869224422442\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.12it/s]\n",
      "3it [00:00, 25.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 39 with Accuracy :0.9017822265625 Loss :17431.1640625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.31it/s]\n",
      "2it [00:00, 17.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 39 with Accuracy :0.654010029909241\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16it [00:00, 17.37it/s]\n",
      "3it [00:00, 25.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Epoch 40 with Accuracy :0.90380859375 Loss :17174.318359375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6it [00:00, 29.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Epoch 40 with Accuracy :0.6629118966584159\n",
      "time equal 44.58629536628723\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "start = time.time()\n",
    "epochs = 40\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    train_accuracy = 0\n",
    "    count = 0\n",
    "    accumulated_loss = 0\n",
    "    for train_in,train_label in tqdm(batch(X_train,Y_train)):\n",
    "        #print(train_in)\n",
    "        torch.cuda.synchronize()\n",
    "        output = model(train_in.to(device))\n",
    "        #print(output.shape)\n",
    "        output = output.squeeze(1)\n",
    "        #print(output.shape)\n",
    "        train_label = train_label.squeeze(-1)\n",
    "        loss = criterion(output,train_label.long().to(device))\n",
    "        with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "            scaled_loss.backward()\n",
    "        #loss.backward()\n",
    "        optimizer.step()\n",
    "        count += 1\n",
    "        train_accuracy += accuracy(output,train_label)\n",
    "        accumulated_loss += scaled_loss.detach() #scaled_loss\n",
    "        optimizer.zero_grad()\n",
    "       \n",
    "    print('Training Epoch {} with Accuracy :{} Loss :{}'.format(epoch+1,train_accuracy/count,accumulated_loss/count))\n",
    "    with torch.no_grad():\n",
    "        test_acc = 0\n",
    "        test_count = 0\n",
    "        for X_t,Y_t in tqdm(batch(X_test,Y_test)):\n",
    "            test_output = model(X_t.to(device))\n",
    "            test_output = test_output.squeeze(1)\n",
    "\n",
    "            Y_t = Y_t.squeeze(-1)\n",
    "            test_accuracy = accuracy(test_output,Y_t)\n",
    "            test_acc += test_accuracy\n",
    "            test_count += 1\n",
    "            \n",
    "    print('Testing Epoch {} with Accuracy :{}'.format(epoch+1,test_acc/test_count))  \n",
    "    \n",
    "end = time.time()\n",
    "print(\"time equal {}\".format(end-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9045737984"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.memory_allocated() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CNN_TC(nn.Module):\n",
    "    def __init__(self,googleNews_vectors,Filters):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding.from_pretrained(googleNews_vectors,freeze=True)\n",
    "        self.embedding.weight.requires_grad = False\n",
    "        self.conv1 = nn.Conv1d(1,1,kernel_size=(Filters[0] * 300),stride=(300))\n",
    "        self.conv2 = nn.Conv1d(1,1,kernel_size=(Filters[1] * 300),stride=(300))\n",
    "        self.conv3 = nn.Conv1d(1,1,kernel_size=(Filters[2] * 300),stride=(300))\n",
    "        \n",
    "        self.max_conv1 = nn.MaxPool1d(kernel_size=(15))\n",
    "        self.max_conv2 = nn.MaxPool1d(kernel_size=(11))\n",
    "        self.max_conv3 = nn.MaxPool1d(kernel_size=(9))\n",
    "        \n",
    "        self.FFN1 = nn.Linear(9,20)\n",
    "        self.FFN2 = nn.Linear(20,2)\n",
    "        \n",
    "    def forward(self,inputs):\n",
    "        X = self.embedding(inputs)\n",
    "        X = X.view(X.shape[0],1,-1).contiguous()\n",
    "\n",
    "        max_feature1 = self.max_conv1(f.relu(self.conv1(X)))\n",
    "        max_feature2 = self.max_conv2(f.relu(self.conv2(X)))\n",
    "        max_feature3 = self.max_conv3(f.relu(self.conv3(X)))\n",
    "        features = torch.cat((max_feature1,max_feature2,max_feature3), -1).squeeze(0)\n",
    "                             \n",
    "        output = self.FFN2(f.relu(self.FFN1(features)))\n",
    "        return output\n",
    "    \n",
    "    \n",
    "model = CNN_TC(googleNews_vectors,Filters=[3,4,5]).to(device)\n",
    "optimizer = torch.optim.Adam(model.parameters())\n",
    "criterion = nn.CrossEntropyLoss().cuda()\n",
    "model, optimizer = amp.initialize(model, optimizer,opt_level='O1')\n",
    "import time\n",
    "start = time.time()\n",
    "epochs = 40\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    train_accuracy = 0\n",
    "    count = 0\n",
    "    accumulated_loss = 0\n",
    "    for train_in,train_label in tqdm(batch(X_train,Y_train)):\n",
    "        #print(train_in)\n",
    "        torch.cuda.synchronize()\n",
    "        output = model(train_in.to(device))\n",
    "        #print(output.shape)\n",
    "        output = output.squeeze(1)\n",
    "        #print(output.shape)\n",
    "        train_label = train_label.squeeze(-1)\n",
    "        loss = criterion(output,train_label.long().to(device))\n",
    "        #with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "            #scaled_loss.backward()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        count += 1\n",
    "        train_accuracy += accuracy(output,train_label)\n",
    "        accumulated_loss += loss#scaled_loss\n",
    "        optimizer.zero_grad()\n",
    "       \n",
    "    print('Training Epoch {} with Accuracy :{} Loss :{}'.format(epoch+1,train_accuracy/count,accumulated_loss/count))\n",
    "    with torch.no_grad():\n",
    "        test_acc = 0\n",
    "        test_count = 0\n",
    "        for X_t,Y_t in tqdm(batch(X_test,Y_test)):\n",
    "            test_output = model(X_t.to(device))\n",
    "            test_output = test_output.squeeze(1)\n",
    "\n",
    "            Y_t = Y_t.squeeze(-1)\n",
    "            test_accuracy = accuracy(test_output,Y_t)\n",
    "            test_acc += test_accuracy\n",
    "            test_count += 1\n",
    "            \n",
    "    print('Testing Epoch {} with Accuracy :{}'.format(epoch+1,test_acc/test_count))  \n",
    "    \n",
    "end = time.time()\n",
    "print(\"time equal {}\".format(end-start))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}