simple_language-model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 55,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2ngQqAVIVPGx",
        "outputId": "14c99874-6e8e-4dea-80a4-d385f6c211d5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10, Loss: 2.274981737136841\n",
            "Epoch 20, Loss: 2.247117757797241\n",
            "Epoch 30, Loss: 2.2201223373413086\n",
            "Epoch 40, Loss: 2.1932365894317627\n",
            "Epoch 50, Loss: 2.1660993099212646\n",
            "Epoch 60, Loss: 2.138852834701538\n",
            "Epoch 70, Loss: 2.1121013164520264\n",
            "Epoch 80, Loss: 2.086674928665161\n",
            "Epoch 90, Loss: 2.063289165496826\n",
            "Epoch 100, Loss: 2.042316198348999\n",
            "Next predicted word: cats\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "\n",
        "# Define the training data\n",
        "corpus = [\n",
        "    \"I love cats\",\n",
        "    \"I adore dogs\",\n",
        "    \"Cats are cute\",\n",
        "    \"Dogs are loyal\"\n",
        "]\n",
        "\n",
        "# Create a vocabulary from the corpus\n",
        "vocab = set(\" \".join(corpus).split())\n",
        "word_to_idx = {word: idx for idx, word in enumerate(vocab)}\n",
        "idx_to_word = {idx: word for word, idx in word_to_idx.items()}\n",
        "\n",
        "# Convert the corpus to training examples\n",
        "train_inputs = []\n",
        "train_targets = []\n",
        "for sentence in corpus:\n",
        "    words = sentence.split()\n",
        "    for i in range(len(words)):\n",
        "        context = words[:i] + words[i+1:]\n",
        "        target = words[i]\n",
        "        train_inputs.append(context)\n",
        "        train_targets.append(target)\n",
        "\n",
        "\n",
        "# Define the neural network model\n",
        "class SimpleLanguageModel(nn.Module):\n",
        "    def __init__(self, vocab_size, embedding_dim, hidden_dim):\n",
        "        super(SimpleLanguageModel, self).__init__()\n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
        "        self.hidden = nn.Linear(embedding_dim, hidden_dim)\n",
        "        self.output = nn.Linear(hidden_dim, vocab_size)\n",
        "        self.softmax = nn.Softmax(dim=1)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        embedded = self.embedding(x)\n",
        "        hidden = self.hidden(embedded)\n",
        "        output = self.output(hidden)\n",
        "        probabilities = self.softmax(output)\n",
        "        return probabilities\n",
        "\n",
        "# Define model hyperparameters\n",
        "vocab_size = len(vocab)\n",
        "embedding_dim = 10\n",
        "hidden_dim = 20\n",
        "\n",
        "# Create an instance of the language model\n",
        "model = SimpleLanguageModel(vocab_size, embedding_dim, hidden_dim)\n",
        "\n",
        "# Define the loss function\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "\n",
        "# Define the optimizer\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.1)\n",
        "\n",
        "# Convert training data to tensor\n",
        "train_inputs = torch.tensor([[word_to_idx[word] for word in context] for context in train_inputs])\n",
        "train_targets = torch.tensor([word_to_idx[word] for word in train_targets])\n",
        "\n",
        "# Repeat each target word twice to match the batch size\n",
        "train_targets = train_targets.repeat(2)\n",
        "\n",
        "# Training loop\n",
        "for epoch in range(100):\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    # Forward pass\n",
        "    output = model(train_inputs)\n",
        "    \n",
        "    # Calculate the loss\n",
        "    loss = loss_function(output.view(-1, vocab_size), train_targets)\n",
        "    \n",
        "    # Backward pass\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "\n",
        "    # Print the loss for every 10 epochs\n",
        "    if (epoch + 1) % 10 == 0:\n",
        "        print(f\"Epoch {epoch+1}, Loss: {loss.item()}\")\n",
        "\n",
        "# Example usage\n",
        "example_input = torch.tensor([word_to_idx[word] for word in [\"I\", \"love\"]])\n",
        "output = model(example_input.unsqueeze(0))\n",
        "predicted_word_idx = torch.argmax(output)\n",
        "\n",
        "predicted_word = idx_to_word[predicted_word_idx.item()]\n",
        "print(f\"Next predicted word: {predicted_word}\")\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# another example: \n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "\n",
        "# Define the training data\n",
        "corpus = [\n",
        "    \"I love cats\",\n",
        "    \"I adore dogs\",\n",
        "    \"Cats are cute\",\n",
        "    \"Dogs are loyal\"\n",
        "]\n",
        "\n",
        "# Create a vocabulary from the corpus\n",
        "vocab = set(\" \".join(corpus).split())\n",
        "word_to_idx = {word: idx for idx, word in enumerate(vocab)}\n",
        "idx_to_word = {idx: word for word, idx in word_to_idx.items()}\n",
        "\n",
        "# Convert the corpus to training examples\n",
        "train_inputs = []\n",
        "train_targets = []\n",
        "for sentence in corpus:\n",
        "    words = sentence.split()\n",
        "    for i in range(len(words) - 1):\n",
        "        train_inputs.append(words[i])\n",
        "        train_targets.append(words[i + 1])\n",
        "print(\"Training inputs:\", train_inputs)\n",
        "print(\"Training targets:\", train_targets)\n",
        "\n",
        "# Define the neural network model\n",
        "class SimpleLanguageModel(nn.Module):\n",
        "    def __init__(self, vocab_size, embedding_dim, hidden_dim):\n",
        "        super(SimpleLanguageModel, self).__init__()\n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
        "        self.hidden = nn.Linear(embedding_dim, hidden_dim)\n",
        "        self.output = nn.Linear(hidden_dim, vocab_size)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        embedded = self.embedding(x)\n",
        "        hidden = self.hidden(embedded)\n",
        "        output = self.output(hidden)\n",
        "        return output\n",
        "\n",
        "# Define model hyperparameters\n",
        "vocab_size = len(vocab)\n",
        "embedding_dim = 10\n",
        "hidden_dim = 20\n",
        "\n",
        "# Create an instance of the language model\n",
        "model = SimpleLanguageModel(vocab_size, embedding_dim, hidden_dim)\n",
        "\n",
        "# Define the loss function\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "\n",
        "# Define the optimizer\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.1)\n",
        "\n",
        "# Convert training data to tensor\n",
        "train_inputs = torch.tensor([word_to_idx[word] for word in train_inputs])\n",
        "train_targets = torch.tensor([word_to_idx[word] for word in train_targets])\n",
        "\n",
        "# Training loop\n",
        "for epoch in range(100):\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    # Forward pass\n",
        "    output = model(train_inputs.unsqueeze(1))\n",
        "    \n",
        "    # Calculate the loss\n",
        "    loss = loss_function(output.view(-1, vocab_size), train_targets)\n",
        "    \n",
        "    # Backward pass\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "\n",
        "    # Print the loss for every 10 epochs\n",
        "    if (epoch + 1) % 10 == 0:\n",
        "        print(f\"Epoch {epoch+1}, Loss: {loss.item()}\")\n",
        "\n",
        "# Example usage\n",
        "example_input = torch.tensor(word_to_idx[\"I\"]).unsqueeze(0)\n",
        "output = model(example_input.unsqueeze(1))\n",
        "predicted_word_idx = torch.argmax(output)\n",
        "predicted_word = idx_to_word[predicted_word_idx.item()]\n",
        "print(f\"Next predicted word: {predicted_word}\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C8rDZ1lsVPo6",
        "outputId": "da79d22b-3e59-40b7-c77d-b661c3c3ceeb"
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training inputs: ['I', 'love', 'I', 'adore', 'Cats', 'are', 'Dogs', 'are']\n",
            "Training targets: ['love', 'cats', 'adore', 'dogs', 'are', 'cute', 'are', 'loyal']\n",
            "Epoch 10, Loss: 1.6497174501419067\n",
            "Epoch 20, Loss: 1.0566790103912354\n",
            "Epoch 30, Loss: 0.7125686407089233\n",
            "Epoch 40, Loss: 0.540465772151947\n",
            "Epoch 50, Loss: 0.4618834853172302\n",
            "Epoch 60, Loss: 0.42338570952415466\n",
            "Epoch 70, Loss: 0.4020962417125702\n",
            "Epoch 80, Loss: 0.3890976309776306\n",
            "Epoch 90, Loss: 0.38053813576698303\n",
            "Epoch 100, Loss: 0.37456631660461426\n",
            "Next predicted word: adore\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ivAb_KGzlUII"
      },
      "execution_count": 58,
      "outputs": []
    }
  ]
}