4Developers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "4Developers.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/CelineBoudier/google-colab-notebooks/blob/main/4Developers.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lBB6SIcIJl68"
      },
      "source": [
        "#**4Developers - code snippets**\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CFGea2dmJxWr"
      },
      "source": [
        "Trigrams, custom\n",
        "\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0EJdoBa_pABb",
        "outputId": "a4c69ccd-35ee-456f-cee2-d9b0d880b265",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "import nltk\n",
        "from collections import Counter\n",
        "from nltk.util import trigrams\n",
        "\n",
        "from nltk.corpus import udhr\n",
        "nltk.download('udhr')\n",
        "languages = [('English', 'Latin1'), ('Polish','Latin2')]\n",
        "\n",
        "def get_trigrams(words):\n",
        "  text_trigrams = []\n",
        "  for word in words:\n",
        "    text_trigrams.extend(get_words_trigrams(word))\n",
        "  return dict(Counter(text_trigrams))\n",
        "\n",
        "def get_words_trigrams(word):\n",
        "  if len(word)>2:\n",
        "    return list(trigrams(word))\n",
        "  else:\n",
        "    return [tuple(word)]\n",
        "    \n",
        "def create_corpus_data(languages):\n",
        "  data_corpus = {}\n",
        "  for lang, enc in languages:\n",
        "    data_corpus[lang] = get_trigrams(udhr.words(f\"{lang}-{enc}\"))\n",
        "  return data_corpus\n",
        "\n",
        "def compute_language_probs(text, data):\n",
        "    text_trigrams = get_trigrams(text.split())\n",
        "    trigrams_number = sum(text_trigrams.values())\n",
        "    probas = {}\n",
        "    for language, trigram_counter in data.items():\n",
        "        prob = 0.0\n",
        "        corpus_trigrams_number = sum(trigram_counter.values())\n",
        "        for k, v in text_trigrams.items():\n",
        "            try:\n",
        "                freq = float(trigram_counter[k])\n",
        "            except KeyError:\n",
        "                freq = 0.0\n",
        "            prob += (freq/float(corpus_trigrams_number)) * (float(v)/ float(trigrams_number))\n",
        "        probas[language]=prob\n",
        "    return(Counter(probas))\n",
        "  \n",
        "    \n",
        "print(compute_language_probs(\"Warszawa\", create_corpus_data(languages)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package udhr to /root/nltk_data...\n",
            "[nltk_data]   Package udhr is already up-to-date!\n",
            "Counter({'Polish': 0.0009128006872852234, 'English': 0.0})\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NeK3FVcmJ1lf"
      },
      "source": [
        "language detection, tools\n",
        "\n",
        "*  https://pypi.org/project/langdetect/\n",
        "*  https://github.com/saffsd/langid.py \n",
        "*  fasttext\n",
        "*  nltk\n",
        "\n",
        "\n",
        "---\n",
        "\n",
        "You need to upload a corpus like lid.176.ftz, available here https://fasttext.cc/docs/en/language-identification.html\n",
        "\n",
        "\n",
        "---\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_PaYjoORNPts",
        "outputId": "77993f4d-5406-405a-b1d2-c4505afbe1c0",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "!pip install -q langdetect\n",
        "!pip install -q langid\n",
        "!pip install -q pyfasttext\n",
        "\n",
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-93beee16-34ee-405f-b8f4-d42ff588018e\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-93beee16-34ee-405f-b8f4-d42ff588018e\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving lid.176.ftz to lid.176.ftz\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uPiS6aSGJ3Kw",
        "outputId": "6b163ae9-2f0e-46db-c7a6-b4255ff152c3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 595
        }
      },
      "source": [
        "sentences = [\"W Szczebrzeszynie chrząszcz brzmi w trzcinie.\",\n",
        "             \"W czasie suszy szosa sucha.\",\n",
        "             \"Awesome news from the lunar floccer championship!\",\n",
        "             \"Ostatnia powieść pana Stegosaurusa jest sukcesem…\",\n",
        "             \"Wow!\",\n",
        "             \"To jest\"]\n",
        "from langdetect import detect\n",
        "\n",
        "from langid import classify\n",
        "\n",
        "import nltk\n",
        "nltk.download('crubadan')\n",
        "nltk.download('punkt')\n",
        "\n",
        "from pyfasttext import FastText\n",
        "model = FastText('lid.176.ftz')\n",
        "\n",
        "for sentence in sentences:\n",
        "  print (sentence)\n",
        "  print(classify(sentence))\n",
        "  print(detect(sentence))\n",
        "  print(nltk.classify.textcat.TextCat().guess_language(sentence))\n",
        "  print(model.predict_proba_single(sentence, k=1))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package crubadan to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/crubadan.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "W Szczebrzeszynie chrząszcz brzmi w trzcinie.\n",
            "('pl', -143.46634340286255)\n",
            "pl\n",
            "pol\n",
            "[('pl', 0.9690467141146767)]\n",
            "W czasie suszy szosa sucha.\n",
            "('pl', -97.29982662200928)\n",
            "pl\n",
            "pol\n",
            "[('pl', 0.9052880878364841)]\n",
            "Awesome news from the lunar floccer championship!\n",
            "('en', -95.2265248298645)\n",
            "en\n",
            "eng \n",
            "[('en', 0.908706667617861)]\n",
            "Ostatnia powieść pana Stegosaurusa jest sukcesem…\n",
            "('pl', -104.08984470367432)\n",
            "pl\n",
            "pol\n",
            "[('pl', 0.9941520616520834)]\n",
            "Wow!\n",
            "('en', 9.061840057373047)\n",
            "pl\n",
            "aka\n",
            "[('en', 0.8432655354756698)]\n",
            "To jest\n",
            "('pl', -10.821922779083252)\n",
            "hr\n",
            "bos\n",
            "[('pl', 1.0)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WmL7gR4HPn1x"
      },
      "source": [
        "Markov chains, custom\n",
        "\n",
        "\n",
        "---\n",
        "You need to upload some texts in .txt format in one language, like some novels. Ex: http://www.gutenberg.org/ebooks/author/32429\n",
        "\n",
        "\n",
        "---\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qFRo1ZnjekDo",
        "outputId": "d70523c6-b47a-4dca-f554-00eaaa481d60",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 62
        }
      },
      "source": [
        "from google.colab import files\n",
        "\n",
        "uploaded = files.upload()\n",
        "\n",
        "def generate_triples(text, gen_words=True):\n",
        "    if gen_words:\n",
        "      words = text.split()\n",
        "    else:\n",
        "      words = [text[i:i+3] for i in range(len(text)-3)]\n",
        "    triples = []\n",
        "    if len(words) >=3:\n",
        "      for i in range(len(words) -2):\n",
        "        triples.append((words[i].replace(\"\\'\", \"'\").replace(\"\\\\\", \"\"), words[i + 1].replace(\"\\'\", \"'\").replace(\"\\\\\", \"\"), words[i + 2].replace(\"\\'\", \"'\").replace(\"\\\\\", \"\")))\n",
        "    return triples\n",
        "      \n",
        "def generate_data(feed_text, gen_words=True):\n",
        "    triples = generate_triples(feed_text, gen_words)\n",
        "    data = {}\n",
        "    for a,b,c in triples:\n",
        "      try:\n",
        "        data[(a, b)].append(c)\n",
        "      except:\n",
        "        data[(a, b)] = [c]\n",
        "    return data   \n",
        " \n",
        "with open(\"mickiewicz.txt\") as feed_file:\n",
        "  feed_text = feed_file.read()\n",
        "data = generate_data(feed_text, False)\n",
        "  "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-0305d993-839b-4bf4-b33e-e79bc0b3b765\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-0305d993-839b-4bf4-b33e-e79bc0b3b765\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qIXcGW3FUBv6",
        "outputId": "89de6bb5-e88f-4c2b-8c1b-dc9003445992",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "import random\n",
        "beginners = [i for i in data.keys() if i[0][0].isupper()]\n",
        "\n",
        "def generate_sentence(min_number_words, gen_words=True):\n",
        "  choose_tuple = random.choice(beginners)\n",
        "  sentence_1 = choose_tuple[0]\n",
        "  sentence_2 = choose_tuple[1]\n",
        "  sentence_3 = random.choice(data[choose_tuple])\n",
        "  if gen_words:\n",
        "    separator = \" \"\n",
        "    sentence_elems = [sentence_1, sentence_2, sentence_3]\n",
        "  else: \n",
        "    separator = \"\"\n",
        "    sentence_elems = [sentence_1, sentence_2[-1], sentence_3[-1]]\n",
        "  while (len(sentence_elems) < min_number_words) or ((len(sentence_elems) >= min_number_words) and not sentence_3.endswith(\".\")and not sentence_3.endswith(\"!\")and not sentence_3.endswith(\"?\")):\n",
        "    sentence_1, sentence_2 = sentence_2, sentence_3\n",
        "    sentence_3 = random.choice(data[(sentence_1, sentence_2)])\n",
        "    if gen_words:\n",
        "      sentence_elems.append(sentence_3)\n",
        "    else:\n",
        "      sentence_elems.append(sentence_3[-1])\n",
        "  generated_sentence = separator.join(sentence_elems)\n",
        "  return generated_sentence\n",
        "\n",
        "\n",
        "generate_sentence(5, False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'A on jej matki;\\n    Śmierć błagamy się pokręgi tobie.'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hT6S4zM3mEHy"
      },
      "source": [
        "Examples:\n",
        "\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "'I? I had brought a few badly lit corridors and waited for a while, until I realized hours had slipped by as it was clearly harmless, not like clothing moths or other bugs.'\n",
        "I let the stations flow past in a modernist museum.\n",
        "Curly, long fingertips brush on silky skin while a shark takes its toll on a derelict path, waiting for the train that would lead me to enter.\n",
        "We danced and we are still the only one to live this incredible experience.\n",
        "'I dziecię bierze do ręki, U łona białego tuli, „Luli, woła, mój maleńki!'\n",
        "\n",
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cTZ0Ce3lm0HZ"
      },
      "source": [
        "get antonyms\n",
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Un1VMzrJmykW",
        "outputId": "2063d9bd-80c7-461f-ec57-e7e8b419d116",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "import nltk\n",
        "from nltk.corpus import wordnet\n",
        "nltk.download(\"wordnet\")\n",
        "\n",
        "conv_pos = {'NOUN':wordnet.NOUN, 'PROPN':wordnet.NOUN, 'ADJ':wordnet.ADJ,'VERB':wordnet.VERB,'ADV':wordnet.ADV}\n",
        "\n",
        "def find_antonym(word, pos):\n",
        "  if pos in conv_pos:\n",
        "    for syn in wordnet.synsets(word, pos=conv_pos[pos]):\n",
        "      for lemma in syn.lemmas():\n",
        "        if lemma.antonyms():\n",
        "          return lemma.antonyms()[0].name()\n",
        "  return word\n",
        "\n",
        "print(find_antonym(\"Hate\", \"NOUN\"))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "love\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0T3QWM6qrQ8R"
      },
      "source": [
        "POS and NER\n",
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8oFvUfulrmbj",
        "outputId": "a54c65a2-53c8-4e4f-f489-e4107df9c788",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 2533
        }
      },
      "source": [
        "import spacy\n",
        "nlp = spacy.load('en_core_web_sm')\n",
        "\n",
        "import nltk\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk import ne_chunk\n",
        "nltk.download('punkt')\n",
        "nltk.download(\"wordnet\")\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "nltk.download('maxent_ne_chunker')\n",
        "nltk.download('words')\n",
        "\n",
        "\n",
        "texts= [\"This is a love song by Queen from the UK.\", \n",
        "        \"Hello, Warsaw!\",\n",
        "        \"Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo.\",\n",
        "        \"Time flies like an arrow.\",\n",
        "        \"Fruit flies like a banana.\"]\n",
        "        \n",
        "for text in texts:\n",
        "  tokens_spacy = nlp(text)\n",
        "  print(text)\n",
        "  print(\"\\n\")\n",
        "  nltk_tok = word_tokenize(text)\n",
        "  tokens_nltk = nltk.pos_tag(nltk_tok)\n",
        "  \n",
        "  print(\"spaCy\")\n",
        "  print(\"POS\")\n",
        "  for token in tokens_spacy:\n",
        "    print(token.text,token.pos_, token.tag_)\n",
        "    \n",
        "  print(\"\\n\")\n",
        "  print(\"NER\")\n",
        "  for ent in tokens_spacy.ents:\n",
        "    print(ent.text, ent.label_)\n",
        "    \n",
        "  print(\"\\nnltk\")\n",
        "  print(ne_chunk(tokens_nltk))\n",
        "  \n",
        "  print(\"\\n\\n\\n\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "[nltk_data] Downloading package maxent_ne_chunker to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping chunkers/maxent_ne_chunker.zip.\n",
            "[nltk_data] Downloading package words to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/words.zip.\n",
            "This is a love song by Queen from the UK.\n",
            "\n",
            "\n",
            "spaCy\n",
            "POS\n",
            "This DET DT\n",
            "is VERB VBZ\n",
            "a DET DT\n",
            "love NOUN NN\n",
            "song NOUN NN\n",
            "by ADP IN\n",
            "Queen PROPN NNP\n",
            "from ADP IN\n",
            "the DET DT\n",
            "UK PROPN NNP\n",
            ". PUNCT .\n",
            "\n",
            "\n",
            "NER\n",
            "UK GPE\n",
            "\n",
            "nltk\n",
            "(S\n",
            "  This/DT\n",
            "  is/VBZ\n",
            "  a/DT\n",
            "  love/NN\n",
            "  song/NN\n",
            "  by/IN\n",
            "  (PERSON Queen/NNP)\n",
            "  from/IN\n",
            "  the/DT\n",
            "  (ORGANIZATION UK/NNP)\n",
            "  ./.)\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Hello, Warsaw!\n",
            "\n",
            "\n",
            "spaCy\n",
            "POS\n",
            "Hello INTJ UH\n",
            ", PUNCT ,\n",
            "Warsaw PROPN NNP\n",
            "! PUNCT .\n",
            "\n",
            "\n",
            "NER\n",
            "Warsaw GPE\n",
            "\n",
            "nltk\n",
            "(S (GPE Hello/NNP) ,/, (PERSON Warsaw/NNP) !/.)\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo.\n",
            "\n",
            "\n",
            "spaCy\n",
            "POS\n",
            "Buffalo PROPN NNP\n",
            "buffalo NOUN NN\n",
            "Buffalo PROPN NNP\n",
            "buffalo NOUN NN\n",
            "buffalo NOUN NN\n",
            "buffalo NOUN NN\n",
            "Buffalo PROPN NNP\n",
            "buffalo NOUN NN\n",
            ". PUNCT .\n",
            "\n",
            "\n",
            "NER\n",
            "Buffalo GPE\n",
            "Buffalo GPE\n",
            "Buffalo GPE\n",
            "\n",
            "nltk\n",
            "(S\n",
            "  (GPE Buffalo/NNP)\n",
            "  buffalo/NN\n",
            "  (PERSON Buffalo/NNP)\n",
            "  buffalo/NN\n",
            "  buffalo/NN\n",
            "  buffalo/NN\n",
            "  (PERSON Buffalo/NNP)\n",
            "  buffalo/NN\n",
            "  ./.)\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Time flies like an arrow.\n",
            "\n",
            "\n",
            "spaCy\n",
            "POS\n",
            "Time PROPN NNP\n",
            "flies VERB VBZ\n",
            "like ADP IN\n",
            "an DET DT\n",
            "arrow NOUN NN\n",
            ". PUNCT .\n",
            "\n",
            "\n",
            "NER\n",
            "\n",
            "nltk\n",
            "(S Time/NNP flies/NNS like/IN an/DT arrow/NN ./.)\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Fruit flies like a banana.\n",
            "\n",
            "\n",
            "spaCy\n",
            "POS\n",
            "Fruit PROPN NNP\n",
            "flies VERB VBZ\n",
            "like ADP IN\n",
            "a DET DT\n",
            "banana NOUN NN\n",
            ". PUNCT .\n",
            "\n",
            "\n",
            "NER\n",
            "\n",
            "nltk\n",
            "(S (GPE Fruit/NNP) flies/VBZ like/IN a/DT banana/NN ./.)\n",
            "\n",
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4UFvAR_mwAwY"
      },
      "source": [
        "sentence stemmer\n",
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CEcVjRs7zowE",
        "outputId": "b2f233c2-fbd4-4032-deb8-53ab2dee3587",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "\n",
        "\n",
        "def stem_word(sentence):\n",
        "  phrase = nlp(sentence)\n",
        "  words = []\n",
        "  for token in phrase:\n",
        "    if token.dep_ not in [\"aux\", \"det\"]:\n",
        "      words.append(token.text)\n",
        "  return \" \".join(words)\n",
        "\n",
        "\n",
        "print(stem_word(\"a small and thinly populated rural area\"))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "small and thinly populated rural area\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6PNMK0AisO9j"
      },
      "source": [
        "antonimiser - spaCy - loop\n",
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pY0_rJKdsX3M",
        "outputId": "ec434451-d647-42ac-d01b-be6dd38880e4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "depth = 3\n",
        "        \n",
        "        \n",
        "def rec_find_antonym(word, pos, depth=1):\n",
        "        antonym = find_antonym(word, pos)\n",
        "        if antonym != word:\n",
        "          return antonym\n",
        "        if depth<=1:\n",
        "            return word\n",
        "        else:\n",
        "            definition = nlp(stem_word(find_definition(word, pos)))\n",
        "            return \" \".join([rec_find_antonym(token.text, token.pos_, depth-1) for token in definition])\n",
        "          \n",
        "def find_definition(word, pos):\n",
        "  if pos in conv_pos:\n",
        "    for syn in wordnet.synsets(word, pos=conv_pos[pos]):\n",
        "      if syn.definition():\n",
        "        return syn.definition().split(\";\")[0]\n",
        "  return word\n",
        "      \n",
        "def antonimize_text(text):\n",
        "  tokens = nlp(text)\n",
        "  antonimized_words = []\n",
        "  for token in tokens:\n",
        "    antonimized_words.append(rec_find_antonym(token.text, token.pos_, depth))\n",
        "  return \" \".join(antonimized_words)\n",
        "\n",
        "print(antonimize_text(\"A simple truth\"))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "A complex falsity\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nKYeZgfUs4x1"
      },
      "source": [
        "Reverse definition lookup\n",
        "\n",
        "\n",
        "---\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_U8JMncLtUKl",
        "outputId": "aaaf3b73-e89c-4743-99c3-a1394b28ca32",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "api = \"https://api.datamuse.com/words?ml=\"\n",
        "\n",
        "import json\n",
        "import requests\n",
        "\n",
        "def get_word_from_def(text):\n",
        "  api_url = api+\"+\".join(text.split())\n",
        "  response = requests.get(api_url)\n",
        "  if len(response.json())>0:\n",
        "    #print (response.json())\n",
        "    return response.json()[0][\"word\"]\n",
        "  else:\n",
        "    return text\n",
        "  \n",
        "print(get_word_from_def(\"capital of poland\"))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "warsaw\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0o1NcmSlwCIz"
      },
      "source": [
        "\n",
        "new antonymiser\n",
        "\n",
        "\n",
        "---\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "adDHqS2IveWF",
        "outputId": "22d18f65-8dba-4920-8008-64fc74481adc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "def rec_find_antonym(word, pos, depth):\n",
        "        antonym = find_antonym(word, pos)\n",
        "        if antonym != word:\n",
        "          return antonym\n",
        "        if depth<=1:\n",
        "            return word\n",
        "        else:\n",
        "            definition = nlp(stem_word(find_definition(word, pos)))\n",
        "            return \" \".join([rec_find_antonym(token.text, token.pos_, depth-1) for token in definition])\n",
        "          \n",
        "def find_definition(word, pos):\n",
        "  if pos in conv_pos:\n",
        "    for syn in wordnet.synsets(word, pos=conv_pos[pos]):\n",
        "      if syn.definition():\n",
        "        return syn.definition().split(\";\")[0]\n",
        "  return word\n",
        "      \n",
        "def antonimize_text(text, depth=3):\n",
        "  tokens = nlp(text)\n",
        "  antonimized_words = []\n",
        "  for token in tokens:\n",
        "    antonym_def = rec_find_antonym(token.text, token.pos_, depth)\n",
        "    if len(antonym_def.split()) > 1:\n",
        "       antonym_def=get_word_from_def(antonym_def)\n",
        "    antonimized_words.append(antonym_def)\n",
        "  return \" \".join(antonimized_words)\n",
        "\n",
        "print(antonimize_text(\"Hello Warsaw\", 2))\n",
        "#print(antonimize_text(\"hand\", 3))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Hello lodz\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UCgQ-J2yv9dP"
      },
      "source": [
        "word embeddings\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m_msgSCNyhi0",
        "outputId": "9c28c410-2856-41f5-b810-a08b84ba29fd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 504
        }
      },
      "source": [
        "import spacy.cli\n",
        "spacy.cli.download(\"en_core_web_lg\")\n",
        "\n",
        "nlp_vec = spacy.load(\"en_core_web_lg\")  \n",
        "\n",
        "def semantic_similarity(word1, word2):\n",
        "  token1 = nlp_vec(word1)\n",
        "  token2 = nlp_vec(word2)\n",
        "  return token1.similarity(token2)\n",
        "\n",
        "print (semantic_similarity(\"life\", \"death\"))\n",
        "\n",
        "\n",
        "#!wget -c http://mattmahoney.net/dc/enwik9.zip -P data\n",
        "#!unzip data/enwik9.zip -d data\n",
        "\n",
        "from pyfasttext import FastText\n",
        "model = FastText('lid.176.ftz')\n",
        "#model.cbow(input='text.txt', output='model', epoch=100, lr=0.7)\n",
        "\n",
        "print(model.most_similar(positive=[\"warsaw\", \"france\"], negative=[\"poland\"], k=1))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\u001b[93m    Linking successful\u001b[0m\n",
            "    /usr/local/lib/python3.6/dist-packages/en_core_web_lg -->\n",
            "    /usr/local/lib/python3.6/dist-packages/spacy/data/en_core_web_lg\n",
            "\n",
            "    You can now load the model via spacy.load('en_core_web_lg')\n",
            "\n",
            "0.5892018368228293\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "ModuleNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-10-d63cb169ede2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;31m#!unzip data/enwik9.zip -d data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mpyfasttext\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mFastText\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFastText\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'lid.176.ftz'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;31m#model.cbow(input='text.txt', output='model', epoch=100, lr=0.7)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'pyfasttext'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ]
        }
      ]
    }
  ]
}