BasicLDAnoSpellCheck.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sparknlp.base import *\n",
    "from sparknlp.annotator import *\n",
    "from sparknlp.pretrained import PretrainedPipeline\n",
    "import sparknlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time \n",
    "t0 = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = sparknlp.start(gpu=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - in-memory</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://192.168.0.13:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.1.1</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>Spark NLP</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x7f290481ed00>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part of Speech"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('datasets/CorpusDemo.csv')\n",
    "df = df[['CONTENT']].drop_duplicates().reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def clean_text(text):\n",
    "    text = re.sub('[#@][^\\t\\n\\r\\f\\v\\s]*', \" \", text)\n",
    "    text = re.sub('[^\\w\\d\\:\\/\\.\\-\\_\\,\\(\\)]', \" \", text)\n",
    "    text = re.sub('(http|www)[^\\t\\n\\r\\f\\v\\s]*', \" \", text)\n",
    "    text = re.sub('\\s+', ' ', text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|             CONTENT|\n",
      "+--------------------+\n",
      "|Los proyectos #Fr...|\n",
      "|Un pequeño pero c...|\n",
      "|Colombia ha debid...|\n",
      "|Es una Mierda #Fr...|\n",
      "|\"El planeta sufre...|\n",
      "|🔶🔸Descargue aqu...|\n",
      "|Pilotos de #frack...|\n",
      "|#IMPORTANTE  Los ...|\n",
      "|Los que creen que...|\n",
      "|Interesante artíc...|\n",
      "+--------------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "text_df = spark.createDataFrame(df)\n",
    "text_df.show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import functions as F\n",
    "\n",
    "udf_clean_text = F.udf(lambda x: clean_text(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_df = text_df.select(udf_clean_text(F.col('CONTENT')).alias('CONTENT'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Custom lemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from es_lemmatizer import lemmatize\n",
    "import spacy\n",
    "\n",
    "nlp = spacy.load(\"es_core_news_sm\")\n",
    "nlp.add_pipe(lemmatize, after=\"tagger\")\n",
    "\n",
    "\n",
    "try:\n",
    "    doc = nlp(''.join(text_df.select('CONTENT').rdd.flatMap(lambda x: x).collect()))\n",
    "except:\n",
    "    doc = nlp(text)\n",
    "\n",
    "custom_lemm = {}\n",
    "for token in doc:\n",
    "    if token.lemma_ not in custom_lemm:\n",
    "        custom_lemm[token.lemma_] = [str(token)]\n",
    "    else:\n",
    "        if str(token) not in custom_lemm[token.lemma_]:\n",
    "            custom_lemm[token.lemma_].append(str(token))\n",
    "        \n",
    "        \n",
    "keys = list(custom_lemm.keys())\n",
    "vals = ['\\t'.join(entry) for entry in list(custom_lemm.values())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "keys[keys.index('q')] = 'que'\n",
    "keys[keys.index('d')] = 'de'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "custom_lemm_list = [f'{key}->{val}\\n' for key, val in zip(keys, vals)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('customLemmanoSpellCheck.txt', 'w') as file:\n",
    "    file.writelines(custom_lemm_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pipeline 1-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "documentAssembler = DocumentAssembler()\\\n",
    "    .setInputCol(\"CONTENT\")\\\n",
    "    .setOutputCol('document')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer()\\\n",
    "    .setInputCols('document')\\\n",
    "    .setOutputCol('token')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmatizer = Lemmatizer()\\\n",
    "    .setInputCols('token')\\\n",
    "    .setOutputCol(\"lemma\")\\\n",
    "    .setDictionary('customLemmanoSpellCheck.txt', '->', '\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import nltk\n",
    "# nltk.download('stopwords')\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "es_stopwords = stopwords.words('spanish')\n",
    "es_stopwords.extend(['de+el', 'a+el'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwordsCleaner = StopWordsCleaner()\\\n",
    "    .setInputCols('lemma')\\\n",
    "    .setOutputCol('1-gram')\\\n",
    "    .setStopWords(es_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "nGrammer = NGramGenerator()\\\n",
    "    .setInputCols('1-gram')\\\n",
    "    .setOutputCol('n-grams')\\\n",
    "    .setN(3)\\\n",
    "    .setEnableCumulative(True)\\\n",
    "    .setDelimiter('_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pos_ud_gsd download started this may take some time.\n",
      "Approximate size to download 5.2 MB\n",
      "[OK!]\n"
     ]
    }
   ],
   "source": [
    "posTagger = PerceptronModel.pretrained(\"pos_ud_gsd\", \"es\")\\\n",
    "    .setInputCols(['document', '1-gram'])\\\n",
    "    .setOutputCol('posTagger')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "finisher = Finisher()\\\n",
    "    .setInputCols(['1-gram', 'n-grams', 'posTagger'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "pipeline = Pipeline() \\\n",
    "    .setStages([documentAssembler,\n",
    "#                 sentenceDetector,\n",
    "                tokenizer,\n",
    "                lemmatizer,\n",
    "                stopwordsCleaner,\n",
    "                nGrammer,\n",
    "                posTagger,\n",
    "                finisher])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_texts = pipeline.fit(text_df).transform(text_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------------+--------------------+--------------------+\n",
      "|             CONTENT|     finished_1-gram|    finished_n-grams|  finished_posTagger|\n",
      "+--------------------+--------------------+--------------------+--------------------+\n",
      "|Los proyectos se ...|[proyecto, ubicar...|[proyecto, ubicar...|[NOUN, VERB, NOUN...|\n",
      "|Un pequeño pero c...|[pequeño, complet...|[pequeño, complet...|[ADJ, ADJ, NOUN, ...|\n",
      "|Colombia ha debid...|[colombia, haber,...|[colombia, haber,...|[PROPN, AUX, VERB...|\n",
      "|      Es una Mierda |            [mierda]|            [mierda]|             [PROPN]|\n",
      "| El planeta sufre...|[planeta, sufrir,...|[planeta, sufrir,...|[NOUN, VERB, NOUN...|\n",
      "| Descargue aquí e...|[descargue, aquí,...|[descargue, aquí,...|[VERB, ADV, NOUN,...|\n",
      "|Pilotos de de Eco...|[piloto, ecopetro...|[piloto, ecopetro...|[NOUN, PROPN, AUX...|\n",
      "| Los pilotos de d...|[piloto, ecopetro...|[piloto, ecopetro...|[NOUN, NOUN, AUX,...|\n",
      "|Los que creen que...|[crear, va, quita...|[crear, va, quita...|[VERB, AUX, VERB,...|\n",
      "|Interesante artíc...|[interesante, art...|[interesante, art...|[ADJ, NOUN, NOUN,...|\n",
      "|Las pequeñas y me...|[pequeño, mediano...|[pequeño, mediano...|[ADJ, NOUN, NOUN,...|\n",
      "|6 años es muy poc...|[6, años, crisis,...|[6, años, crisis,...|[NUM, NOUN, NOUN,...|\n",
      "| El planeta sufre...|[planeta, sufrir,...|[planeta, sufrir,...|[NOUN, VERB, NOUN...|\n",
      "| Duque puso como ...|[duque, poner, mi...|[duque, poner, mi...|[NOUN, VERB, NOUN...|\n",
      "| El planeta sufre...|[planeta, sufrir,...|[planeta, sufrir,...|[NOUN, VERB, NOUN...|\n",
      "|PRONUNCIAMIENTO E...|[pronunciamiento,...|[pronunciamiento,...|[NOUN, ADJ, ADJ, ...|\n",
      "|Se les.olvida que...|[les.olvida, inve...|[les.olvida, inve...|[VERB, VERB, ADV,...|\n",
      "|PRONUNCIAMIENTO E...|[pronunciamiento,...|[pronunciamiento,...|[NOUN, ADJ, ADJ, ...|\n",
      "| de pronto el del...|     [pronto, hacer]|[pronto, hacer, p...|         [ADV, VERB]|\n",
      "|Todos dedicados a...|[dedicados, covid...|[dedicados, covid...|[VERB, PROPN, PUN...|\n",
      "+--------------------+--------------------+--------------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "processed_texts.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Handling n-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import functions as F\n",
    "from pyspark.sql import types as T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# User defined function to join the list\n",
    "udf_join_arr = F.udf(lambda x: ' '.join(x), T.StringType())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_texts = processed_texts.withColumn('finished_posTagger',  udf_join_arr(F.col('finished_posTagger')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------------+--------------------+--------------------+\n",
      "|             CONTENT|     finished_1-gram|    finished_n-grams|  finished_posTagger|\n",
      "+--------------------+--------------------+--------------------+--------------------+\n",
      "|Los proyectos se ...|[proyecto, ubicar...|[proyecto, ubicar...|NOUN VERB NOUN NO...|\n",
      "|Un pequeño pero c...|[pequeño, complet...|[pequeño, complet...|ADJ ADJ NOUN NOUN...|\n",
      "|Colombia ha debid...|[colombia, haber,...|[colombia, haber,...|PROPN AUX VERB VE...|\n",
      "|      Es una Mierda |            [mierda]|            [mierda]|               PROPN|\n",
      "| El planeta sufre...|[planeta, sufrir,...|[planeta, sufrir,...|NOUN VERB NOUN AD...|\n",
      "| Descargue aquí e...|[descargue, aquí,...|[descargue, aquí,...|VERB ADV NOUN NOU...|\n",
      "|Pilotos de de Eco...|[piloto, ecopetro...|[piloto, ecopetro...|NOUN PROPN AUX VE...|\n",
      "| Los pilotos de d...|[piloto, ecopetro...|[piloto, ecopetro...|NOUN NOUN AUX VER...|\n",
      "|Los que creen que...|[crear, va, quita...|[crear, va, quita...|VERB AUX VERB NOU...|\n",
      "|Interesante artíc...|[interesante, art...|[interesante, art...|ADJ NOUN NOUN PRO...|\n",
      "|Las pequeñas y me...|[pequeño, mediano...|[pequeño, mediano...|ADJ NOUN NOUN NOU...|\n",
      "|6 años es muy poc...|[6, años, crisis,...|[6, años, crisis,...|NUM NOUN NOUN ADJ...|\n",
      "| El planeta sufre...|[planeta, sufrir,...|[planeta, sufrir,...|NOUN VERB NOUN AD...|\n",
      "| Duque puso como ...|[duque, poner, mi...|[duque, poner, mi...|NOUN VERB NOUN AD...|\n",
      "| El planeta sufre...|[planeta, sufrir,...|[planeta, sufrir,...|NOUN VERB NOUN AD...|\n",
      "|PRONUNCIAMIENTO E...|[pronunciamiento,...|[pronunciamiento,...|NOUN ADJ ADJ NOUN...|\n",
      "|Se les.olvida que...|[les.olvida, inve...|[les.olvida, inve...|VERB VERB ADV NOU...|\n",
      "|PRONUNCIAMIENTO E...|[pronunciamiento,...|[pronunciamiento,...|NOUN ADJ ADJ NOUN...|\n",
      "| de pronto el del...|     [pronto, hacer]|[pronto, hacer, p...|            ADV VERB|\n",
      "|Todos dedicados a...|[dedicados, covid...|[dedicados, covid...|VERB PROPN PUNCT ...|\n",
      "+--------------------+--------------------+--------------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "processed_texts.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "posDocumentAssembler = DocumentAssembler()\\\n",
    "    .setInputCol('finished_posTagger')\\\n",
    "    .setOutputCol('pos_document')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "posTokenizer = Tokenizer()\\\n",
    "    .setInputCols('pos_document')\\\n",
    "    .setOutputCol('pos')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "posNGrammer = NGramGenerator()\\\n",
    "    .setInputCols('pos')\\\n",
    "    .setOutputCol('pos_ngrams')\\\n",
    "    .setN(3)\\\n",
    "    .setEnableCumulative(True)\\\n",
    "    .setDelimiter('_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "posFinisher = Finisher()\\\n",
    "    .setInputCols(['pos', 'pos_ngrams'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "posPipeline = Pipeline()\\\n",
    "    .setStages([posDocumentAssembler,\n",
    "                posTokenizer,\n",
    "                posNGrammer,\n",
    "                posFinisher])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_texts = posPipeline.fit(processed_texts).transform(processed_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n",
      "|             CONTENT|     finished_1-gram|    finished_n-grams|  finished_posTagger|        finished_pos| finished_pos_ngrams|\n",
      "+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n",
      "|Los proyectos se ...|[proyecto, ubicar...|[proyecto, ubicar...|NOUN VERB NOUN NO...|[NOUN, VERB, NOUN...|[NOUN, VERB, NOUN...|\n",
      "|Un pequeño pero c...|[pequeño, complet...|[pequeño, complet...|ADJ ADJ NOUN NOUN...|[ADJ, ADJ, NOUN, ...|[ADJ, ADJ, NOUN, ...|\n",
      "|Colombia ha debid...|[colombia, haber,...|[colombia, haber,...|PROPN AUX VERB VE...|[PROPN, AUX, VERB...|[PROPN, AUX, VERB...|\n",
      "|      Es una Mierda |            [mierda]|            [mierda]|               PROPN|             [PROPN]|             [PROPN]|\n",
      "| El planeta sufre...|[planeta, sufrir,...|[planeta, sufrir,...|NOUN VERB NOUN AD...|[NOUN, VERB, NOUN...|[NOUN, VERB, NOUN...|\n",
      "| Descargue aquí e...|[descargue, aquí,...|[descargue, aquí,...|VERB ADV NOUN NOU...|[VERB, ADV, NOUN,...|[VERB, ADV, NOUN,...|\n",
      "|Pilotos de de Eco...|[piloto, ecopetro...|[piloto, ecopetro...|NOUN PROPN AUX VE...|[NOUN, PROPN, AUX...|[NOUN, PROPN, AUX...|\n",
      "| Los pilotos de d...|[piloto, ecopetro...|[piloto, ecopetro...|NOUN NOUN AUX VER...|[NOUN, NOUN, AUX,...|[NOUN, NOUN, AUX,...|\n",
      "|Los que creen que...|[crear, va, quita...|[crear, va, quita...|VERB AUX VERB NOU...|[VERB, AUX, VERB,...|[VERB, AUX, VERB,...|\n",
      "|Interesante artíc...|[interesante, art...|[interesante, art...|ADJ NOUN NOUN PRO...|[ADJ, NOUN, NOUN,...|[ADJ, NOUN, NOUN,...|\n",
      "|Las pequeñas y me...|[pequeño, mediano...|[pequeño, mediano...|ADJ NOUN NOUN NOU...|[ADJ, NOUN, NOUN,...|[ADJ, NOUN, NOUN,...|\n",
      "|6 años es muy poc...|[6, años, crisis,...|[6, años, crisis,...|NUM NOUN NOUN ADJ...|[NUM, NOUN, NOUN,...|[NUM, NOUN, NOUN,...|\n",
      "| El planeta sufre...|[planeta, sufrir,...|[planeta, sufrir,...|NOUN VERB NOUN AD...|[NOUN, VERB, NOUN...|[NOUN, VERB, NOUN...|\n",
      "| Duque puso como ...|[duque, poner, mi...|[duque, poner, mi...|NOUN VERB NOUN AD...|[NOUN, VERB, NOUN...|[NOUN, VERB, NOUN...|\n",
      "| El planeta sufre...|[planeta, sufrir,...|[planeta, sufrir,...|NOUN VERB NOUN AD...|[NOUN, VERB, NOUN...|[NOUN, VERB, NOUN...|\n",
      "|PRONUNCIAMIENTO E...|[pronunciamiento,...|[pronunciamiento,...|NOUN ADJ ADJ NOUN...|[NOUN, ADJ, ADJ, ...|[NOUN, ADJ, ADJ, ...|\n",
      "|Se les.olvida que...|[les.olvida, inve...|[les.olvida, inve...|VERB VERB ADV NOU...|[VERB, VERB, ADV,...|[VERB, VERB, ADV,...|\n",
      "|PRONUNCIAMIENTO E...|[pronunciamiento,...|[pronunciamiento,...|NOUN ADJ ADJ NOUN...|[NOUN, ADJ, ADJ, ...|[NOUN, ADJ, ADJ, ...|\n",
      "| de pronto el del...|     [pronto, hacer]|[pronto, hacer, p...|            ADV VERB|         [ADV, VERB]|[ADV, VERB, ADV_V...|\n",
      "|Todos dedicados a...|[dedicados, covid...|[dedicados, covid...|VERB PROPN PUNCT ...|[VERB, PROPN, PUN...|[VERB, PROPN, PUN...|\n",
      "+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "processed_texts.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filtering POSTags\n",
    "\n",
    "These tags mark the core part-of-speech categories.\n",
    "\n",
    "__Alphabetical listing__:\n",
    "- ADJ: adjective (noun modifiers)\n",
    "- ADP: adposition (preposiciones y postposiciones, e.g., in, to, during)\n",
    "- ADV: adverb (verb -sometines also adjective- modifiers)\n",
    "- AUX: auxiliary \n",
    "- CCONJ: coorinating conjuction (links words without subordination)\n",
    "- DET: determiner\n",
    "- INTJ: interjection\n",
    "- NOUN: noun\n",
    "- NUM: numeral\n",
    "- PART: particle\n",
    "- PRON: pronoun\n",
    "- PROPN: proper noun\n",
    "- PUNCT: punctuation\n",
    "- SCONJ: subordinating conjunction\n",
    "- SYM: symbol\n",
    "- VERB: verb\n",
    "- X: other\n",
    "\n",
    "Reference: https://universaldependencies.org/u/pos/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Filtering 1-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "allowed_cats = ['NUM', 'ADJ', 'NOUN', 'PROPN', 'VERB', 'ADV', 'X']\n",
    "\n",
    "def filter_pos(words, pos_tags):\n",
    "    return [word for word, pos in zip(words, pos_tags) if pos in allowed_cats]\n",
    "\n",
    "udf_filter_pos = F.udf(filter_pos, T.ArrayType(T.StringType()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_texts = processed_texts.withColumn('filtered_1-gram', udf_filter_pos(F.col('finished_1-gram'), F.col('finished_pos')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n",
      "|             CONTENT|     finished_1-gram|    finished_n-grams|  finished_posTagger|        finished_pos| finished_pos_ngrams|     filtered_1-gram|\n",
      "+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n",
      "|Los proyectos se ...|[proyecto, ubicar...|[proyecto, ubicar...|NOUN VERB NOUN NO...|[NOUN, VERB, NOUN...|[NOUN, VERB, NOUN...|[proyecto, ubicar...|\n",
      "|Un pequeño pero c...|[pequeño, complet...|[pequeño, complet...|ADJ ADJ NOUN NOUN...|[ADJ, ADJ, NOUN, ...|[ADJ, ADJ, NOUN, ...|[pequeño, complet...|\n",
      "|Colombia ha debid...|[colombia, haber,...|[colombia, haber,...|PROPN AUX VERB VE...|[PROPN, AUX, VERB...|[PROPN, AUX, VERB...|[colombia, debido...|\n",
      "|      Es una Mierda |            [mierda]|            [mierda]|               PROPN|             [PROPN]|             [PROPN]|            [mierda]|\n",
      "| El planeta sufre...|[planeta, sufrir,...|[planeta, sufrir,...|NOUN VERB NOUN AD...|[NOUN, VERB, NOUN...|[NOUN, VERB, NOUN...|[planeta, sufrir,...|\n",
      "| Descargue aquí e...|[descargue, aquí,...|[descargue, aquí,...|VERB ADV NOUN NOU...|[VERB, ADV, NOUN,...|[VERB, ADV, NOUN,...|[descargue, aquí,...|\n",
      "|Pilotos de de Eco...|[piloto, ecopetro...|[piloto, ecopetro...|NOUN PROPN AUX VE...|[NOUN, PROPN, AUX...|[NOUN, PROPN, AUX...|[piloto, ecopetro...|\n",
      "| Los pilotos de d...|[piloto, ecopetro...|[piloto, ecopetro...|NOUN NOUN AUX VER...|[NOUN, NOUN, AUX,...|[NOUN, NOUN, AUX,...|[piloto, ecopetro...|\n",
      "|Los que creen que...|[crear, va, quita...|[crear, va, quita...|VERB AUX VERB NOU...|[VERB, AUX, VERB,...|[VERB, AUX, VERB,...|[crear, quitar, p...|\n",
      "|Interesante artíc...|[interesante, art...|[interesante, art...|ADJ NOUN NOUN PRO...|[ADJ, NOUN, NOUN,...|[ADJ, NOUN, NOUN,...|[interesante, art...|\n",
      "|Las pequeñas y me...|[pequeño, mediano...|[pequeño, mediano...|ADJ NOUN NOUN NOU...|[ADJ, NOUN, NOUN,...|[ADJ, NOUN, NOUN,...|[pequeño, mediano...|\n",
      "|6 años es muy poc...|[6, años, crisis,...|[6, años, crisis,...|NUM NOUN NOUN ADJ...|[NUM, NOUN, NOUN,...|[NUM, NOUN, NOUN,...|[6, años, crisis,...|\n",
      "| El planeta sufre...|[planeta, sufrir,...|[planeta, sufrir,...|NOUN VERB NOUN AD...|[NOUN, VERB, NOUN...|[NOUN, VERB, NOUN...|[planeta, sufrir,...|\n",
      "| Duque puso como ...|[duque, poner, mi...|[duque, poner, mi...|NOUN VERB NOUN AD...|[NOUN, VERB, NOUN...|[NOUN, VERB, NOUN...|[duque, poner, mi...|\n",
      "| El planeta sufre...|[planeta, sufrir,...|[planeta, sufrir,...|NOUN VERB NOUN AD...|[NOUN, VERB, NOUN...|[NOUN, VERB, NOUN...|[planeta, sufrir,...|\n",
      "|PRONUNCIAMIENTO E...|[pronunciamiento,...|[pronunciamiento,...|NOUN ADJ ADJ NOUN...|[NOUN, ADJ, ADJ, ...|[NOUN, ADJ, ADJ, ...|[pronunciamiento,...|\n",
      "|Se les.olvida que...|[les.olvida, inve...|[les.olvida, inve...|VERB VERB ADV NOU...|[VERB, VERB, ADV,...|[VERB, VERB, ADV,...|[les.olvida, inve...|\n",
      "|PRONUNCIAMIENTO E...|[pronunciamiento,...|[pronunciamiento,...|NOUN ADJ ADJ NOUN...|[NOUN, ADJ, ADJ, ...|[NOUN, ADJ, ADJ, ...|[pronunciamiento,...|\n",
      "| de pronto el del...|     [pronto, hacer]|[pronto, hacer, p...|            ADV VERB|         [ADV, VERB]|[ADV, VERB, ADV_V...|     [pronto, hacer]|\n",
      "|Todos dedicados a...|[dedicados, covid...|[dedicados, covid...|VERB PROPN PUNCT ...|[VERB, PROPN, PUN...|[VERB, PROPN, PUN...|[dedicados, covid...|\n",
      "+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "processed_texts.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Filtering n-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add punctuation\n",
    "\n",
    "filter_3 = ['NUM_ADV_VERB', 'NUM_ADV_ADJ', 'NUM_ADJ_NOUN', 'NUM_NOUN_VERB', 'NUM_NOUN_ADJ',\\\n",
    "            'PROPN_PROPN_PROPN', 'PROPN_VERB_PROPN', 'PROPN_VERB_NOUN', 'PROPN_VERB_ADV', 'PROPN_ADJ_VERB', 'PROPN_ADV_ADJ', 'PROPN_ADV_VERB',\\\n",
    "            'NOUN_PROPN_PROPN', 'NOUM_VERB_NOUN', 'NOUN_VERB_PROPN', 'NOUN_PROPN_VERB', 'NOUM_VERB_NUM', 'NOUN_NUM_VERB', 'NOUN_VERB_ADV',\\\n",
    "            'VERB_ADJ_PROPN', 'VERB_ADJ_NOUN', 'VERB_PROPN_PROPN', 'VERB_NOUN_NOUN', 'VERB_NOUN_PROPN', 'VERB_NOUN_ADJ', 'VERB_PROPN_NOUN', 'VERB_NUM_NOUN', 'VERB_ADV_ADV',\\\n",
    "            'ADJ_NOUN_VERB', 'ADJ_PROPN_VERB', 'ADJ_PROPN_PROPN', 'ADJ_NOUN_PROPN', 'ADJ_PROPN_NOUN', 'ADJ_NOUN_NOUN', 'ADJ_NOUN_ADJ', 'ADJ_PROPN_ADJ', 'ADJ_VERB_NOUN',\\\n",
    "            'ADV_VERB_PROPN', 'ADV_VERB_NOUN']\n",
    "filter_2 = ['PROPN_PROPN', 'PROPN_NOUN', 'PROPN_ADJ', 'PROPN_VERB',\\\n",
    "            'NOUN_NOUN', 'NOUN_PROPN', 'NOUN_VERB', 'NOUN_ADJ',\\\n",
    "            'NUM_NOUN',\\\n",
    "            'ADJ_NOUN', 'ADJ_PROPN',\\\n",
    "            'ADV_VERB', 'ADV_ADJ',\\\n",
    "            'VERB_NOUN', 'VERB_PROPN', 'VERB_ADV', 'VERB_ADJ']\n",
    "\n",
    "def filter_pos_ngrams(words, pos_tags):\n",
    "    return [word for word, pos in zip(words, pos_tags)\\\n",
    "            if (len(pos.split('_')) == 3\\\n",
    "                and\\\n",
    "               pos in filter_3)\\\n",
    "           or (len(pos.split('_')) == 2\\\n",
    "              and\\\n",
    "              pos in filter_2)]\n",
    "\n",
    "udf_filter_pos_combs = F.udf(filter_pos_ngrams, T.ArrayType(T.StringType()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n",
      "|             CONTENT|     finished_1-gram|    finished_n-grams|  finished_posTagger|        finished_pos| finished_pos_ngrams|     filtered_1-gram|     filtered_ngrams|\n",
      "+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n",
      "|Los proyectos se ...|[proyecto, ubicar...|[proyecto, ubicar...|NOUN VERB NOUN NO...|[NOUN, VERB, NOUN...|[NOUN, VERB, NOUN...|[proyecto, ubicar...|[proyecto_ubicar,...|\n",
      "|Un pequeño pero c...|[pequeño, complet...|[pequeño, complet...|ADJ ADJ NOUN NOUN...|[ADJ, ADJ, NOUN, ...|[ADJ, ADJ, NOUN, ...|[pequeño, complet...|[completo_análisi...|\n",
      "|Colombia ha debid...|[colombia, haber,...|[colombia, haber,...|PROPN AUX VERB VE...|[PROPN, AUX, VERB...|[PROPN, AUX, VERB...|[colombia, debido...|[7_años, existir_...|\n",
      "|      Es una Mierda |            [mierda]|            [mierda]|               PROPN|             [PROPN]|             [PROPN]|            [mierda]|                  []|\n",
      "| El planeta sufre...|[planeta, sufrir,...|[planeta, sufrir,...|NOUN VERB NOUN AD...|[NOUN, VERB, NOUN...|[NOUN, VERB, NOUN...|[planeta, sufrir,...|[planeta_sufrir, ...|\n",
      "| Descargue aquí e...|[descargue, aquí,...|[descargue, aquí,...|VERB ADV NOUN NOU...|[VERB, ADV, NOUN,...|[VERB, ADV, NOUN,...|[descargue, aquí,...|[descargue_aquí, ...|\n",
      "|Pilotos de de Eco...|[piloto, ecopetro...|[piloto, ecopetro...|NOUN PROPN AUX VE...|[NOUN, PROPN, AUX...|[NOUN, PROPN, AUX...|[piloto, ecopetro...|[piloto_ecopetrol...|\n",
      "| Los pilotos de d...|[piloto, ecopetro...|[piloto, ecopetro...|NOUN NOUN AUX VER...|[NOUN, NOUN, AUX,...|[NOUN, NOUN, AUX,...|[piloto, ecopetro...|[piloto_ecopetrol...|\n",
      "|Los que creen que...|[crear, va, quita...|[crear, va, quita...|VERB AUX VERB NOU...|[VERB, AUX, VERB,...|[VERB, AUX, VERB,...|[crear, quitar, p...|[quitar_privacida...|\n",
      "|Interesante artíc...|[interesante, art...|[interesante, art...|ADJ NOUN NOUN PRO...|[ADJ, NOUN, NOUN,...|[ADJ, NOUN, NOUN,...|[interesante, art...|[interesante_artí...|\n",
      "|Las pequeñas y me...|[pequeño, mediano...|[pequeño, mediano...|ADJ NOUN NOUN NOU...|[ADJ, NOUN, NOUN,...|[ADJ, NOUN, NOUN,...|[pequeño, mediano...|[pequeño_mediano,...|\n",
      "|6 años es muy poc...|[6, años, crisis,...|[6, años, crisis,...|NUM NOUN NOUN ADJ...|[NUM, NOUN, NOUN,...|[NUM, NOUN, NOUN,...|[6, años, crisis,...|[6_años, años_cri...|\n",
      "| El planeta sufre...|[planeta, sufrir,...|[planeta, sufrir,...|NOUN VERB NOUN AD...|[NOUN, VERB, NOUN...|[NOUN, VERB, NOUN...|[planeta, sufrir,...|[planeta_sufrir, ...|\n",
      "| Duque puso como ...|[duque, poner, mi...|[duque, poner, mi...|NOUN VERB NOUN AD...|[NOUN, VERB, NOUN...|[NOUN, VERB, NOUN...|[duque, poner, mi...|[duque_poner, pon...|\n",
      "| El planeta sufre...|[planeta, sufrir,...|[planeta, sufrir,...|NOUN VERB NOUN AD...|[NOUN, VERB, NOUN...|[NOUN, VERB, NOUN...|[planeta, sufrir,...|[planeta_sufrir, ...|\n",
      "|PRONUNCIAMIENTO E...|[pronunciamiento,...|[pronunciamiento,...|NOUN ADJ ADJ NOUN...|[NOUN, ADJ, ADJ, ...|[NOUN, ADJ, ADJ, ...|[pronunciamiento,...|[pronunciamiento_...|\n",
      "|Se les.olvida que...|[les.olvida, inve...|[les.olvida, inve...|VERB VERB ADV NOU...|[VERB, VERB, ADV,...|[VERB, VERB, ADV,...|[les.olvida, inve...|[invertir_ahora, ...|\n",
      "|PRONUNCIAMIENTO E...|[pronunciamiento,...|[pronunciamiento,...|NOUN ADJ ADJ NOUN...|[NOUN, ADJ, ADJ, ...|[NOUN, ADJ, ADJ, ...|[pronunciamiento,...|[pronunciamiento_...|\n",
      "| de pronto el del...|     [pronto, hacer]|[pronto, hacer, p...|            ADV VERB|         [ADV, VERB]|[ADV, VERB, ADV_V...|     [pronto, hacer]|      [pronto_hacer]|\n",
      "|Todos dedicados a...|[dedicados, covid...|[dedicados, covid...|VERB PROPN PUNCT ...|[VERB, PROPN, PUN...|[VERB, PROPN, PUN...|[dedicados, covid...|[dedicados_covid_...|\n",
      "+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "processed_texts = processed_texts.withColumn('filtered_ngrams',udf_filter_pos_combs(F.col('finished_n-grams'), F.col('finished_pos_ngrams')))\n",
    "processed_texts.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('proyecto', 'NOUN'),\n",
       " ('ubicar', 'VERB'),\n",
       " ('desembocadura', 'NOUN'),\n",
       " ('río', 'NOUN'),\n",
       " ('sogamoso', 'PROPN'),\n",
       " (',', 'PUNCT'),\n",
       " ('2', 'NUM'),\n",
       " ('recurso', 'NOUN'),\n",
       " ('importante', 'ADJ'),\n",
       " ('tener', 'VERB'),\n",
       " ('zona', 'NOUN'),\n",
       " ('humedal', 'ADJ'),\n",
       " ('caño', 'ADJ'),\n",
       " ('interconectar', 'VERB'),\n",
       " ('ríos', 'PROPN'),\n",
       " ('ciénaga', 'PROPN'),\n",
       " ('aun', 'SCONJ'),\n",
       " ('habitar', 'VERB'),\n",
       " ('manatí', 'ADV')]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list([(x, y) for x,y in zip(processed_texts.select('finished_1-gram').toPandas()['finished_1-gram'].tolist()[0], processed_texts.select('finished_posTagger').toPandas()['finished_posTagger'].str.split(' ').tolist()[0])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# processed_texts.select('filtered_ngrams').toPandas().filtered_ngrams.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Joiner and Vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import concat\n",
    "\n",
    "processed_texts = processed_texts.withColumn('final', concat(F.col('filtered_1-gram'), F.col('filtered_ngrams')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TF: Term Frequency\n",
    "\n",
    "from pyspark.ml.feature import CountVectorizer\n",
    "\n",
    "tfizer = CountVectorizer(inputCol='final',\\\n",
    "                         outputCol='tf_features')\n",
    "tf_model = tfizer.fit(processed_texts)\n",
    "tf_result = tf_model.transform(processed_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tf_result.select('tf_features').toPandas().tf_features.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# IDF: Inverse Document Frequency\n",
    "\n",
    "from pyspark.ml.feature import IDF\n",
    "\n",
    "idfizer = IDF(inputCol='tf_features',\\\n",
    "              outputCol='tf_idf_features')\n",
    "tfidf_result = idfizer.fit(tf_result).transform(tf_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tfidf_result.select('tf_idf_features').toPandas().tf_idf_features.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.clustering import LDA\n",
    "\n",
    "num_topics = 6\n",
    "max_iter = 10\n",
    "\n",
    "lda = LDA(k=num_topics, maxIter=max_iter, featuresCol='tf_idf_features', seed=24)\n",
    "lda_model = lda.fit(tfidf_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = tf_model.vocabulary\n",
    "\n",
    "def get_words(token_list):\n",
    "    return [vocab[token_id] for token_id in token_list]\n",
    "\n",
    "udf_to_words = F.udf(get_words, T.ArrayType(T.StringType()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+--------------------+\n",
      "|topic|          topicWords|\n",
      "+-----+--------------------+\n",
      "|    0|[hacer, agua, pet...|\n",
      "|    1|[recurso, favor, ...|\n",
      "|    2|[social, licencia...|\n",
      "|    3|[ley, convenciona...|\n",
      "|    4|[fracking, ecopet...|\n",
      "|    5|[piloto, presiden...|\n",
      "+-----+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "num_top_words = 150\n",
    "\n",
    "\n",
    "topics = lda_model.describeTopics(num_top_words).withColumn('topicWords', udf_to_words(F.col('termIndices')))\n",
    "topics.select('topic', 'topicWords').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+--------------------+--------------------+\n",
      "|topic|         termIndices|         termWeights|\n",
      "+-----+--------------------+--------------------+\n",
      "|    0|[3, 5, 15, 1, 2, ...|[0.00201077417876...|\n",
      "|    1|[43, 11, 136, 123...|[0.00172384546465...|\n",
      "|    2|[41, 233, 105, 52...|[0.00152803965447...|\n",
      "|    3|[20, 110, 17, 39,...|[0.00141001902481...|\n",
      "|    4|[2, 103, 142, 4, ...|[0.00130768684606...|\n",
      "|    5|[1, 31, 21, 48, 0...|[0.00178757147594...|\n",
      "+-----+--------------------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lda_model.describeTopics().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['recurso',\n",
       " 'favor',\n",
       " 'territorios',\n",
       " '2',\n",
       " 'gobierno',\n",
       " 'menos',\n",
       " 'buscar',\n",
       " 'vida',\n",
       " 'país',\n",
       " 'hacer',\n",
       " 'venir',\n",
       " 'proteger',\n",
       " 'defender',\n",
       " 'colombiano',\n",
       " 'solo',\n",
       " 'pueblo',\n",
       " 'votar',\n",
       " 'pasar',\n",
       " 'vivir',\n",
       " 'dar',\n",
       " 'decir',\n",
       " 'recurso_natural',\n",
       " 'querer',\n",
       " 'ecosistemas',\n",
       " 'natural',\n",
       " 'justificar',\n",
       " 'cosa',\n",
       " 'audiencia',\n",
       " 'artículo',\n",
       " 'representante',\n",
       " 'dejar',\n",
       " 'matar',\n",
       " 'tal',\n",
       " 'territorio',\n",
       " 'campaña',\n",
       " 'verdad',\n",
       " 'aprobar',\n",
       " 'cambio',\n",
       " '1',\n",
       " 'mas',\n",
       " 'medio',\n",
       " 'magdalena',\n",
       " 'senador',\n",
       " 'ciudadanía',\n",
       " 'proyecto',\n",
       " 'mensaje',\n",
       " 'cámara',\n",
       " 'empresa',\n",
       " 'hoy',\n",
       " 'fuente',\n",
       " 'páramo',\n",
       " 'ley',\n",
       " 'negocio',\n",
       " 'economía',\n",
       " 'votar_favor',\n",
       " 'crear',\n",
       " 'contar',\n",
       " 'nefasto',\n",
       " 'multinacional',\n",
       " 'megaminería',\n",
       " 'audiencia_público',\n",
       " 'colombia',\n",
       " 'apoyar',\n",
       " 'amenazar',\n",
       " 'información',\n",
       " 'tributario',\n",
       " 'voto',\n",
       " 'aprobación',\n",
       " 'contrato',\n",
       " 'onu',\n",
       " '3',\n",
       " 'cesar',\n",
       " 'departamento',\n",
       " 'gobernar',\n",
       " 'impacto',\n",
       " 'apoyo',\n",
       " 'promesa',\n",
       " 'encima',\n",
       " 'fracking',\n",
       " 'nuevo',\n",
       " 'foto',\n",
       " 'técnico',\n",
       " 'nacional',\n",
       " 'piloto',\n",
       " 'importancia',\n",
       " 'representar',\n",
       " 'bravo',\n",
       " 'importante',\n",
       " 'salud',\n",
       " 'destrucción',\n",
       " 'pedir',\n",
       " 'ambiental',\n",
       " 'producir',\n",
       " 'parar',\n",
       " 'noche',\n",
       " 'plenaria',\n",
       " 'análisis',\n",
       " 'actividad',\n",
       " 'incentivo',\n",
       " 'deuda',\n",
       " 'hídrico',\n",
       " 'compartir',\n",
       " 'vital',\n",
       " 'comunidades',\n",
       " 'impacto_ambiental',\n",
       " 'hundir',\n",
       " 'directo',\n",
       " 'secar',\n",
       " 'surtir',\n",
       " 'permitir',\n",
       " 'senador_votar',\n",
       " 'demandada',\n",
       " '12',\n",
       " 'magdalena_medio',\n",
       " 'aportar',\n",
       " 'lista',\n",
       " 'mal',\n",
       " 'congreso',\n",
       " 'revivir',\n",
       " 'oportunidad',\n",
       " 'llamado',\n",
       " 'presidente',\n",
       " 'proposición',\n",
       " 'parte',\n",
       " '80',\n",
       " 'insiste',\n",
       " 'mirada',\n",
       " '210',\n",
       " 'saber',\n",
       " 'santander_magdalena',\n",
       " 'secar_megaminería_páramo',\n",
       " 'santander',\n",
       " '80_hogares',\n",
       " 'medio_recurso_vital',\n",
       " 'recurso_vital',\n",
       " 'hogares',\n",
       " 'medio_recurso',\n",
       " 'surtir_agua',\n",
       " 'hogares_país',\n",
       " 'secar_megaminería',\n",
       " 'modelo',\n",
       " 'megaminería_páramo',\n",
       " 'región',\n",
       " 'palabra',\n",
       " 'páramo_surtir',\n",
       " 'positivo_favor',\n",
       " 'presión',\n",
       " 'protegerlo',\n",
       " 'valer',\n",
       " 'kalé']"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topics.toPandas().iloc[1]['topicWords']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stop Spark Context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39.14262104034424"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time.time()- t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:nlp]",
   "language": "python",
   "name": "conda-env-nlp-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}