Yelp_Top_Words_for_Each_Business_RA.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNsXWm/sqgtvMilskwqLlon",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/EAName/yelp-reviews/blob/main/Yelp_Top_Words_for_Each_Business_RA.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "92Dg33FmuxQs"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import os\n",
        "import random\n",
        "import numpy as np\n",
        "import pickle \n",
        "import re,string\n",
        "\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem import PorterStemmer\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.tokenize import word_tokenize \n",
        "\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.cluster import KMeans, AgglomerativeClustering\n",
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "from sklearn.decomposition import TruncatedSVD, LatentDirichletAllocation\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.metrics import roc_auc_score, accuracy_score, confusion_matrix, silhouette_score\n",
        "from sklearn.metrics import precision_score, recall_score, f1_score, classification_report\n",
        "from sklearn.metrics import mean_squared_error as MSE\n",
        "from sklearn.metrics import mean_absolute_error\n",
        "from sklearn.model_selection import cross_val_score, StratifiedKFold\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.linear_model import LinearRegression\n",
        "from sklearn.ensemble import RandomForestRegressor\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.model_selection import train_test_split, KFold\n",
        "from sklearn.manifold import MDS\n",
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "from sklearn.ensemble import GradientBoostingClassifier\n",
        "from sklearn.svm import LinearSVC\n",
        "\n",
        "import xgboost as xgb\n",
        "\n",
        "from sklearn.model_selection import train_test_split \n",
        "from sklearn.model_selection import GridSearchCV \n",
        "from sklearn.model_selection import cross_val_score \n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from sklearn.model_selection import KFold \n",
        "from sklearn.model_selection import cross_val_predict\n",
        "\n",
        "\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from gensim.models import Word2Vec,LdaMulticore, TfidfModel, CoherenceModel\n",
        "from gensim import corpora\n",
        "from gensim import similarities\n",
        "from gensim.models.doc2vec import Doc2Vec, TaggedDocument\n",
        "from gensim.models import LsiModel,LdaModel\n",
        "\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "#from keras.preprocessing.sequence import pad_sequences\n",
        "from tqdm import tqdm\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Embedding, LSTM, Dense, SpatialDropout1D, GRU\n",
        "from keras.callbacks import EarlyStopping\n",
        "from keras.initializers import Constant\n",
        "from sklearn.model_selection import train_test_split\n",
        "from tensorflow.keras.optimizers import Adam, Nadam, RMSprop\n",
        "from keras import Input\n",
        "\n",
        "from scipy import sparse\n",
        "\n",
        "from IPython.display import display"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('omw-1.4')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VGHly-zIpris",
        "outputId": "37fb01f1-970b-41e2-eb83-e7f197ac99fd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ],
      "metadata": {
        "id": "ovbC3RlUpxKS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xqTk3PNGvHH2",
        "outputId": "3a1bf1e9-d23e-4137-f4da-340df3e1bcce"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4l7SoL-w5nVa"
      },
      "source": [
        "## Load Reviews Dataset"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df =pd.read_csv('/content/gdrive/MyDrive/Data/yelp_reviews_clean_version4_st_louis.csv')"
      ],
      "metadata": {
        "id": "p2KLuijTcWil"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['stars'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pl7B9M89zjpr",
        "outputId": "f65b6dc2-57db-410c-b0f4-e631c085e79a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "5.0    124247\n",
              "4.0     73648\n",
              "3.0     35345\n",
              "1.0     32873\n",
              "2.0     26263\n",
              "Name: stars, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = df[df['stars'].isin([1.0, 4.0, 5.0])] "
      ],
      "metadata": {
        "id": "Mq0iqK58zo9o"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['stars'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c2O7BPUKz9Qe",
        "outputId": "a8071c43-dbb8-427e-ceba-3fb0012b8f4b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "5.0    124247\n",
              "4.0     73648\n",
              "1.0     32873\n",
              "Name: stars, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def clean_doc(doc): \n",
        "    #split review into individual words\n",
        "    tokens=doc.split()\n",
        "    re_punc = re.compile('[%s]' % re.escape(string.punctuation))\n",
        "    # remove punctuation from each word\n",
        "    tokens = [re_punc.sub('', w) for w in tokens]\n",
        "\n",
        "    #word lemmzatization    \n",
        "    lemm = WordNetLemmatizer()\n",
        "    tokens=[lemm.lemmatize(word) for word in tokens]\n",
        "\n",
        "    # remove remaining tokens that are not alphabetic\n",
        "    tokens = [word for word in tokens if word.isalpha()]\n",
        "    # filter out short tokens\n",
        "    tokens = [word for word in tokens if len(word) > 3]\n",
        "    #lowercase all words\n",
        "    tokens = [word.lower() for word in tokens]\n",
        "    # filter out stop words\n",
        "    stop_words = set(stopwords.words('english'))\n",
        "    tokens = [w for w in tokens if not w in stop_words]     \n",
        "\n",
        "    # filter out common words\n",
        "    tokens = [w for w in tokens if not w in ['louis', 'restaurant', 'also', 'youre']] \n",
        "    # # word stemming    \n",
        "    # ps=PorterStemmer()\n",
        "    # tokens=[ps.stem(word) for word in tokens]\n",
        "\n",
        "    return tokens\n",
        "\n",
        "def final_processed_text(doc):\n",
        "    #this is a function to join the processed text back\n",
        "    ' '.join(doc)\n",
        "    return doc"
      ],
      "metadata": {
        "id": "9aA2s_9J1hao"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['cleaned_text'] = df['text'].apply(lambda x: clean_doc(x))\n",
        "\n",
        "final_processed_text = [' '.join(x) for x in df['cleaned_text'].tolist()]\n",
        "df['final_processed_text'] = final_processed_text     #add to dataframe"
      ],
      "metadata": {
        "id": "cFEYh7_b1hc0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Top Negative Words"
      ],
      "metadata": {
        "id": "kMHkaxIIGOUr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_negative = df[df['stars'].isin([1.0])] "
      ],
      "metadata": {
        "id": "UgstTUGL07co"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(df_negative)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n7M-zjzz1JHK",
        "outputId": "dbc6b315-6aa2-4694-a9f6-a4823cee38d2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "32873"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "titles = list(range(len(df_negative)))\n",
        "cv = CountVectorizer(binary=False, ngram_range = (1,1), max_features = 2000)\n",
        "count_matrix = cv.fit_transform(df_negative['final_processed_text'])\n",
        "\n",
        "words = cv.get_feature_names()\n",
        "matrix=pd.DataFrame(count_matrix.toarray(), columns=words, index=titles)"
      ],
      "metadata": {
        "id": "nwXVAw1X07fI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_negative['business_id'].nunique()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GXQrvKja5LGV",
        "outputId": "75b0b2e7-fc81-4fde-dfd7-ddf8537c592f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2859"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# add business_id to count_matrix\n",
        "business_negative = df_negative['business_id'].tolist()\n",
        "matrix['business_id'] = business_negative"
      ],
      "metadata": {
        "id": "H9j_5TKA5RxH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# sum of words for each business\n",
        "df_negative_sum = matrix.groupby('business_id').sum()"
      ],
      "metadata": {
        "id": "XWBUE0oH13y7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(df_negative_sum)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EhHjPjNbEHJz",
        "outputId": "df03a206-4f54-4b20-b90f-8376afb04564"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2859"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "business_id_list = []\n",
        "sentiment_list = []\n",
        "words_list = []\n",
        "count_list = []\n",
        "\n",
        "for index, row in df_negative_sum.iterrows():\n",
        "  sorted_words = row.sort_values(ascending=False)\n",
        "  # print(sorted_words)\n",
        "  # print(sorted_words[0])\n",
        "  # print(sorted_words.index[0])\n",
        "  for i in range(1,16):\n",
        "    business_id_list.append(index)\n",
        "    sentiment_list.append('negative')\n",
        "    words_list.append(sorted_words.index[i])\n",
        "    count_list.append(sorted_words[i])"
      ],
      "metadata": {
        "id": "mGQLKqLISe96"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_negative_top_words = pd.DataFrame(list(zip(business_id_list, sentiment_list, words_list, count_list)),\n",
        "               columns =['business_id', 'sentiment', 'word', 'count'])"
      ],
      "metadata": {
        "id": "RtiQ8Y1MYYHK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_negative_top_words"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "o3W3Ml7SYBNs",
        "outputId": "e9b7119a-e7d8-4c19-a593-f8980fa056eb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                  business_id sentiment     word  count\n",
              "0      -1wnOZFsvsJPEXh_hWcmMA  negative  ordered      2\n",
              "1      -1wnOZFsvsJPEXh_hWcmMA  negative   server      2\n",
              "2      -1wnOZFsvsJPEXh_hWcmMA  negative  finally      2\n",
              "3      -1wnOZFsvsJPEXh_hWcmMA  negative     food      2\n",
              "4      -1wnOZFsvsJPEXh_hWcmMA  negative   minute      1\n",
              "...                       ...       ...      ...    ...\n",
              "42880  zymvTQ12tRy3bzmP4tPgOg  negative     said      8\n",
              "42881  zymvTQ12tRy3bzmP4tPgOg  negative    asked      7\n",
              "42882  zymvTQ12tRy3bzmP4tPgOg  negative   charge      7\n",
              "42883  zymvTQ12tRy3bzmP4tPgOg  negative   minute      7\n",
              "42884  zymvTQ12tRy3bzmP4tPgOg  negative    would      7\n",
              "\n",
              "[42885 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6afa8680-4e39-476d-9871-4c2f549c087e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>business_id</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>word</th>\n",
              "      <th>count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1wnOZFsvsJPEXh_hWcmMA</td>\n",
              "      <td>negative</td>\n",
              "      <td>ordered</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-1wnOZFsvsJPEXh_hWcmMA</td>\n",
              "      <td>negative</td>\n",
              "      <td>server</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-1wnOZFsvsJPEXh_hWcmMA</td>\n",
              "      <td>negative</td>\n",
              "      <td>finally</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-1wnOZFsvsJPEXh_hWcmMA</td>\n",
              "      <td>negative</td>\n",
              "      <td>food</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>-1wnOZFsvsJPEXh_hWcmMA</td>\n",
              "      <td>negative</td>\n",
              "      <td>minute</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42880</th>\n",
              "      <td>zymvTQ12tRy3bzmP4tPgOg</td>\n",
              "      <td>negative</td>\n",
              "      <td>said</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42881</th>\n",
              "      <td>zymvTQ12tRy3bzmP4tPgOg</td>\n",
              "      <td>negative</td>\n",
              "      <td>asked</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42882</th>\n",
              "      <td>zymvTQ12tRy3bzmP4tPgOg</td>\n",
              "      <td>negative</td>\n",
              "      <td>charge</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42883</th>\n",
              "      <td>zymvTQ12tRy3bzmP4tPgOg</td>\n",
              "      <td>negative</td>\n",
              "      <td>minute</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42884</th>\n",
              "      <td>zymvTQ12tRy3bzmP4tPgOg</td>\n",
              "      <td>negative</td>\n",
              "      <td>would</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>42885 rows × 4 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6afa8680-4e39-476d-9871-4c2f549c087e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6afa8680-4e39-476d-9871-4c2f549c087e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6afa8680-4e39-476d-9871-4c2f549c087e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Top Positive Words"
      ],
      "metadata": {
        "id": "b5v0uwnIGSo1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df = df[df['stars'].isin([4.0, 5.0])] "
      ],
      "metadata": {
        "id": "PIS3NnrI4y6G"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "27e4726f-2402-454f-c240-670f6befc510",
        "id": "4l2iawVRGUBk"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "197895"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "titles = list(range(len(df)))\n",
        "cv = CountVectorizer(binary=False, ngram_range = (1,1), max_features = 2000)\n",
        "count_matrix = cv.fit_transform(df['final_processed_text'])\n",
        "\n",
        "words = cv.get_feature_names()\n",
        "matrix=pd.DataFrame(count_matrix.toarray(), columns=words, index=titles)"
      ],
      "metadata": {
        "id": "IfieUtMbGUBl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['business_id'].nunique()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "68de14aa-e484-4c80-d64b-be6987e0274b",
        "id": "YPeXLZpCGUBl"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3163"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# add business_id to count_matrix\n",
        "business_positive = df['business_id'].tolist()\n",
        "matrix['business_id'] = business_positive"
      ],
      "metadata": {
        "id": "il8ybzuHGUBl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# sum of words for each business\n",
        "df_positive_sum = matrix.groupby('business_id').sum()"
      ],
      "metadata": {
        "id": "xgoLaiwKGUBl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(df_positive_sum)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e745e5ca-178f-4c45-f39c-b9ef8ff1a356",
        "id": "A0aRvkUoGUBl"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3163"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for index, row in df_positive_sum.iterrows():\n",
        "  sorted_words = row.sort_values(ascending=False)\n",
        "  # print(sorted_words)\n",
        "  # print(sorted_words[0])\n",
        "  # print(sorted_words.index[0])\n",
        "  for i in range(1,16):\n",
        "    business_id_list.append(index)\n",
        "    sentiment_list.append('positive')\n",
        "    words_list.append(sorted_words.index[i])\n",
        "    count_list.append(sorted_words[i])"
      ],
      "metadata": {
        "id": "lDME4uJ7ZEuW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_top_words = pd.DataFrame(list(zip(business_id_list, sentiment_list, words_list, count_list)),\n",
        "               columns =['business_id', 'sentiment', 'word', 'count'])"
      ],
      "metadata": {
        "id": "gvMGgYzDZEuX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_top_words"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "outputId": "68c77ad0-14f3-436b-ca07-d6168e98732e",
        "id": "xvT9_CtbZEuX"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                  business_id sentiment       word  count\n",
              "0      -1wnOZFsvsJPEXh_hWcmMA  negative    ordered      2\n",
              "1      -1wnOZFsvsJPEXh_hWcmMA  negative     server      2\n",
              "2      -1wnOZFsvsJPEXh_hWcmMA  negative    finally      2\n",
              "3      -1wnOZFsvsJPEXh_hWcmMA  negative       food      2\n",
              "4      -1wnOZFsvsJPEXh_hWcmMA  negative     minute      1\n",
              "...                       ...       ...        ...    ...\n",
              "90325  zzO2zgfqP9ANmEWt-EZFWg  positive    husband      9\n",
              "90326  zzO2zgfqP9ANmEWt-EZFWg  positive      salad      9\n",
              "90327  zzO2zgfqP9ANmEWt-EZFWg  positive       time      9\n",
              "90328  zzO2zgfqP9ANmEWt-EZFWg  positive  perfectly      9\n",
              "90329  zzO2zgfqP9ANmEWt-EZFWg  positive     flavor      8\n",
              "\n",
              "[90330 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f0b224e1-2ea0-418e-9717-ecdc48cbd1b8\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>business_id</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>word</th>\n",
              "      <th>count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1wnOZFsvsJPEXh_hWcmMA</td>\n",
              "      <td>negative</td>\n",
              "      <td>ordered</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-1wnOZFsvsJPEXh_hWcmMA</td>\n",
              "      <td>negative</td>\n",
              "      <td>server</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-1wnOZFsvsJPEXh_hWcmMA</td>\n",
              "      <td>negative</td>\n",
              "      <td>finally</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-1wnOZFsvsJPEXh_hWcmMA</td>\n",
              "      <td>negative</td>\n",
              "      <td>food</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>-1wnOZFsvsJPEXh_hWcmMA</td>\n",
              "      <td>negative</td>\n",
              "      <td>minute</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>90325</th>\n",
              "      <td>zzO2zgfqP9ANmEWt-EZFWg</td>\n",
              "      <td>positive</td>\n",
              "      <td>husband</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>90326</th>\n",
              "      <td>zzO2zgfqP9ANmEWt-EZFWg</td>\n",
              "      <td>positive</td>\n",
              "      <td>salad</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>90327</th>\n",
              "      <td>zzO2zgfqP9ANmEWt-EZFWg</td>\n",
              "      <td>positive</td>\n",
              "      <td>time</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>90328</th>\n",
              "      <td>zzO2zgfqP9ANmEWt-EZFWg</td>\n",
              "      <td>positive</td>\n",
              "      <td>perfectly</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>90329</th>\n",
              "      <td>zzO2zgfqP9ANmEWt-EZFWg</td>\n",
              "      <td>positive</td>\n",
              "      <td>flavor</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>90330 rows × 4 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f0b224e1-2ea0-418e-9717-ecdc48cbd1b8')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f0b224e1-2ea0-418e-9717-ecdc48cbd1b8 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f0b224e1-2ea0-418e-9717-ecdc48cbd1b8');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_top_words.isnull().sum()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zBNOQOIcZfgJ",
        "outputId": "8b950c64-85b3-4549-cd43-d48f032671f0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "business_id    0\n",
              "sentiment      0\n",
              "word           0\n",
              "count          0\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_top_words.to_csv('/content/gdrive/MyDrive/Data/top_words_by_business_v2.csv', index = False)"
      ],
      "metadata": {
        "id": "Q0eu8tDPZ6zS"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}