ExploreDataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Till now, bad performance on: I-ORG (59%), I-MISC (43%), I-TIME (44%), L-TIME (50%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'resources' + os.sep + 'X_test_with_predictions.csv'    \n",
    "preds_df = pd.read_csv(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes_without_O = ['U-PERCENT', 'L-PERS', 'U-PERS', 'L-ORG', 'L-LOC', 'I-ORG', 'I-LOC', 'B-ORG', 'L-DATE', 'I-MONEY', 'B-MISC', 'L-MISC', 'L-MONEY', 'B-LOC', 'B-PERS', 'I-PERS', 'U-DATE', 'B-DATE', 'U-LOC', 'B-MONEY', 'U-MISC', 'I-MISC', 'I-DATE', 'L-PERCENT', 'I-TIME', 'U-ORG', 'L-TIME', 'B-PERCENT', 'B-TIME', 'U-TIME', 'I-PERCENT', 'U-MONEY' ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = list(preds_df['y_pred'].values)\n",
    "y_true = list(preds_df['y_test'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\PC-1\\AppData\\Local\\conda\\conda\\envs\\NLPConda36\\lib\\site-packages\\sklearn\\metrics\\classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\PC-1\\AppData\\Local\\conda\\conda\\envs\\NLPConda36\\lib\\site-packages\\sklearn\\metrics\\classification.py:1145: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "y_true = pd.Series(y_true)\n",
    "y_pred = pd.Series(y_pred)\n",
    "cross_tab = pd.crosstab(y_true, y_pred, rownames=['Real Label'], colnames=['Prediction'], margins=True)\n",
    "report = classification_report(y_true, y_pred, labels=classes_without_O, target_names=classes_without_O)\n",
    "report_with_O = classification_report(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "   U-PERCENT       0.78      0.76      0.77        37\n",
      "      L-PERS       0.89      0.90      0.89       139\n",
      "      U-PERS       0.83      0.81      0.82       131\n",
      "       L-ORG       0.80      0.64      0.71       137\n",
      "       L-LOC       0.68      0.67      0.67        66\n",
      "       I-ORG       0.78      0.48      0.60        60\n",
      "       I-LOC       0.55      0.55      0.55        31\n",
      "       B-ORG       0.70      0.63      0.66       114\n",
      "      L-DATE       0.84      0.75      0.79        36\n",
      "     I-MONEY       0.97      0.97      0.97        37\n",
      "      B-MISC       0.56      0.26      0.36        19\n",
      "      L-MISC       0.62      0.17      0.27        29\n",
      "     L-MONEY       0.98      1.00      0.99        43\n",
      "       B-LOC       0.56      0.69      0.62        61\n",
      "      B-PERS       0.91      0.92      0.91       137\n",
      "      I-PERS       0.73      0.58      0.64        33\n",
      "      U-DATE       0.69      0.69      0.69        32\n",
      "      B-DATE       0.74      0.69      0.72        42\n",
      "       U-LOC       0.85      0.88      0.86       170\n",
      "     B-MONEY       0.87      0.96      0.91        27\n",
      "      U-MISC       0.91      0.85      0.88       126\n",
      "      I-MISC       0.83      0.33      0.48        15\n",
      "      I-DATE       0.61      0.58      0.60        24\n",
      "   L-PERCENT       0.82      0.90      0.86        10\n",
      "      I-TIME       1.00      1.00      1.00         2\n",
      "       U-ORG       0.84      0.67      0.75        79\n",
      "      L-TIME       0.00      0.00      0.00         4\n",
      "   B-PERCENT       0.92      0.86      0.89        14\n",
      "      B-TIME       1.00      0.67      0.80         3\n",
      "      U-TIME       1.00      0.50      0.67         2\n",
      "   I-PERCENT       0.00      0.00      0.00         0\n",
      "     U-MONEY       1.00      0.20      0.33         5\n",
      "\n",
      "   micro avg       0.81      0.75      0.78      1665\n",
      "   macro avg       0.76      0.64      0.68      1665\n",
      "weighted avg       0.81      0.75      0.77      1665\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simplify_labels(y_data):\n",
    "    transformed_y_data = []\n",
    "    for label in y_data:\n",
    "        if len(label.split(\"-\"))>1:\n",
    "            label = label.split(\"-\")[1]\n",
    "        transformed_y_data.append(label)\n",
    "    return transformed_y_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "transformed_y_true = []\n",
    "for label in y_true:\n",
    "    if len(label.split(\"-\"))>1:\n",
    "        label = label.split(\"-\")[1]\n",
    "    transformed_y_true.append(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "transformed_y_pred = []\n",
    "for label in y_pred:\n",
    "    if len(label.split(\"-\"))>1:\n",
    "        label = label.split(\"-\")[1]\n",
    "    transformed_y_pred.append(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_without_O = list(set(list(y_true)))\n",
    "tags_without_O.remove('O')\n",
    "y_true = pd.Series(transformed_y_true)\n",
    "y_pred = pd.Series(transformed_y_pred)\n",
    "cross_tab = pd.crosstab(y_true, y_pred, rownames=['Real Label'], colnames=['Prediction'], margins=True)\n",
    "report = classification_report(y_true, y_pred, labels=tags_without_O, target_names=tags_without_O)\n",
    "report_with_O = classification_report(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "        DATE       0.80      0.75      0.78       134\n",
      "        MISC       0.91      0.68      0.78       189\n",
      "       MONEY       0.97      0.97      0.97       112\n",
      "        TIME       1.00      0.45      0.62        11\n",
      "        PERS       0.91      0.90      0.90       440\n",
      "     PERCENT       0.87      0.85      0.86        61\n",
      "         LOC       0.80      0.84      0.82       328\n",
      "         ORG       0.83      0.67      0.74       390\n",
      "\n",
      "   micro avg       0.86      0.80      0.83      1665\n",
      "   macro avg       0.89      0.76      0.81      1665\n",
      "weighted avg       0.87      0.80      0.83      1665\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(12583, 14)\n",
      "(525, 14)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Number</th>\n",
       "      <th>Person</th>\n",
       "      <th>Pos</th>\n",
       "      <th>Prefix</th>\n",
       "      <th>Status</th>\n",
       "      <th>Suffix</th>\n",
       "      <th>Tense</th>\n",
       "      <th>Token</th>\n",
       "      <th>TokenOrder</th>\n",
       "      <th>Word</th>\n",
       "      <th>y_pred</th>\n",
       "      <th>y_test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>unspecified</td>\n",
       "      <td>כדה\"א</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>כדה\"א</td>\n",
       "      <td>51838</td>\n",
       "      <td>כדה\"א</td>\n",
       "      <td>U-PERS</td>\n",
       "      <td>U-LOC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>feminine</td>\n",
       "      <td>יהודי</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>adjective</td>\n",
       "      <td>ה</td>\n",
       "      <td>absolute</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>היהודיות</td>\n",
       "      <td>37675</td>\n",
       "      <td>היהודיות</td>\n",
       "      <td>U-MISC</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>unspecified</td>\n",
       "      <td>פולין</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>פולין</td>\n",
       "      <td>57340</td>\n",
       "      <td>פולין</td>\n",
       "      <td>U-LOC</td>\n",
       "      <td>L-ORG</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Gender  Lemma       Number       Person         Pos   Prefix  \\\n",
       "44  unspecified  כדה\"א  unspecified  unspecified  properName  no_pref   \n",
       "54     feminine  יהודי       plural  unspecified   adjective        ה   \n",
       "80  unspecified  פולין  unspecified  unspecified  properName  no_pref   \n",
       "\n",
       "         Status     Suffix        Tense     Token  TokenOrder      Word  \\\n",
       "44  unspecified  no_suffix  unspecified     כדה\"א       51838     כדה\"א   \n",
       "54     absolute  no_suffix  unspecified  היהודיות       37675  היהודיות   \n",
       "80  unspecified  no_suffix  unspecified     פולין       57340     פולין   \n",
       "\n",
       "    y_pred y_test  \n",
       "44  U-PERS  U-LOC  \n",
       "54  U-MISC      O  \n",
       "80   U-LOC  L-ORG  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(preds_df.shape)\n",
    "df_with_errors_only = preds_df.query('y_pred != y_test')\n",
    "print(df_with_errors_only.shape)\n",
    "df_with_errors_only.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "O            103\n",
       "L-ORG         50\n",
       "B-ORG         42\n",
       "I-ORG         31\n",
       "U-ORG         26\n",
       "U-PERS        25\n",
       "L-MISC        24\n",
       "L-LOC         22\n",
       "U-LOC         21\n",
       "B-LOC         19\n",
       "U-MISC        19\n",
       "L-PERS        14\n",
       "I-PERS        14\n",
       "B-MISC        14\n",
       "I-LOC         14\n",
       "B-DATE        13\n",
       "B-PERS        11\n",
       "U-DATE        10\n",
       "I-DATE        10\n",
       "I-MISC        10\n",
       "U-PERCENT      9\n",
       "L-DATE         9\n",
       "U-MONEY        4\n",
       "L-TIME         4\n",
       "B-PERCENT      2\n",
       "L-PERCENT      1\n",
       "B-TIME         1\n",
       "U-TIME         1\n",
       "B-MONEY        1\n",
       "I-MONEY        1\n",
       "Name: y_test, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_with_errors_only['y_test'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Number</th>\n",
       "      <th>Person</th>\n",
       "      <th>Pos</th>\n",
       "      <th>Prefix</th>\n",
       "      <th>Status</th>\n",
       "      <th>Suffix</th>\n",
       "      <th>Tense</th>\n",
       "      <th>Token</th>\n",
       "      <th>TokenOrder</th>\n",
       "      <th>Word</th>\n",
       "      <th>y_pred</th>\n",
       "      <th>y_test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>925</th>\n",
       "      <td>feminine</td>\n",
       "      <td>שנה</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>construct</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>שנות</td>\n",
       "      <td>31020</td>\n",
       "      <td>שנות</td>\n",
       "      <td>B-DATE</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3906</th>\n",
       "      <td>unspecified</td>\n",
       "      <td>על</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>preposition</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>על</td>\n",
       "      <td>29771</td>\n",
       "      <td>על</td>\n",
       "      <td>O</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4119</th>\n",
       "      <td>masculine</td>\n",
       "      <td>סרט</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>ה</td>\n",
       "      <td>absolute</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>הסרטים</td>\n",
       "      <td>30400</td>\n",
       "      <td>הסרטים</td>\n",
       "      <td>O</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7307</th>\n",
       "      <td>unspecified</td>\n",
       "      <td>של</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>shel-preposition</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>של</td>\n",
       "      <td>31369</td>\n",
       "      <td>של</td>\n",
       "      <td>O</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7399</th>\n",
       "      <td>masculine</td>\n",
       "      <td>גן</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>absolute</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>גן</td>\n",
       "      <td>30627</td>\n",
       "      <td>גן</td>\n",
       "      <td>I-LOC</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9044</th>\n",
       "      <td>masculine</td>\n",
       "      <td>עולם</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>ה</td>\n",
       "      <td>absolute</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>העולם</td>\n",
       "      <td>39621</td>\n",
       "      <td>העולם</td>\n",
       "      <td>O</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9108</th>\n",
       "      <td>unspecified</td>\n",
       "      <td>של</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>shel-preposition</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>של</td>\n",
       "      <td>30303</td>\n",
       "      <td>של</td>\n",
       "      <td>O</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9533</th>\n",
       "      <td>unspecified</td>\n",
       "      <td>סימון</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>סימון</td>\n",
       "      <td>25504</td>\n",
       "      <td>סימון</td>\n",
       "      <td>L-LOC</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10665</th>\n",
       "      <td>feminine</td>\n",
       "      <td>עלייה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>construct</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>עלית</td>\n",
       "      <td>15114</td>\n",
       "      <td>עלית</td>\n",
       "      <td>O</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12359</th>\n",
       "      <td>masculine</td>\n",
       "      <td>רחוב</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>ב</td>\n",
       "      <td>absolute</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>ברחוב</td>\n",
       "      <td>11404</td>\n",
       "      <td>רחוב</td>\n",
       "      <td>O</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Gender  Lemma       Number       Person               Pos  \\\n",
       "925       feminine    שנה       plural  unspecified              noun   \n",
       "3906   unspecified     על  unspecified  unspecified       preposition   \n",
       "4119     masculine    סרט       plural  unspecified              noun   \n",
       "7307   unspecified     של  unspecified  unspecified  shel-preposition   \n",
       "7399     masculine     גן     singular  unspecified              noun   \n",
       "9044     masculine   עולם     singular  unspecified              noun   \n",
       "9108   unspecified     של  unspecified  unspecified  shel-preposition   \n",
       "9533   unspecified  סימון  unspecified  unspecified        properName   \n",
       "10665     feminine  עלייה     singular  unspecified              noun   \n",
       "12359    masculine   רחוב     singular  unspecified              noun   \n",
       "\n",
       "        Prefix       Status     Suffix        Tense   Token  TokenOrder  \\\n",
       "925    no_pref    construct  no_suffix  unspecified    שנות       31020   \n",
       "3906   no_pref  unspecified  no_suffix  unspecified      על       29771   \n",
       "4119         ה     absolute  no_suffix  unspecified  הסרטים       30400   \n",
       "7307   no_pref  unspecified  no_suffix  unspecified      של       31369   \n",
       "7399   no_pref     absolute  no_suffix  unspecified      גן       30627   \n",
       "9044         ה     absolute  no_suffix  unspecified   העולם       39621   \n",
       "9108   no_pref  unspecified  no_suffix  unspecified      של       30303   \n",
       "9533   no_pref  unspecified  no_suffix  unspecified   סימון       25504   \n",
       "10665  no_pref    construct  no_suffix  unspecified    עלית       15114   \n",
       "12359        ב     absolute  no_suffix  unspecified   ברחוב       11404   \n",
       "\n",
       "         Word  y_pred  y_test  \n",
       "925      שנות  B-DATE  I-MISC  \n",
       "3906       על       O  I-MISC  \n",
       "4119   הסרטים       O  I-MISC  \n",
       "7307       של       O  I-MISC  \n",
       "7399       גן   I-LOC  I-MISC  \n",
       "9044    העולם       O  I-MISC  \n",
       "9108       של       O  I-MISC  \n",
       "9533    סימון   L-LOC  I-MISC  \n",
       "10665    עלית       O  I-MISC  \n",
       "12359    רחוב       O  I-MISC  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_with_errors_only.query('y_test == \"I-MISC\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'resources' + os.sep + 'dataset_biluo.csv'    \n",
    "df = pd.read_csv(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "O            54541\n",
       "U-LOC          829\n",
       "U-MISC         735\n",
       "U-PERS         663\n",
       "L-PERS         662\n",
       "B-PERS         662\n",
       "B-ORG          646\n",
       "L-ORG          646\n",
       "U-ORG          455\n",
       "L-LOC          344\n",
       "B-LOC          344\n",
       "I-ORG          267\n",
       "U-DATE         203\n",
       "B-MONEY        189\n",
       "L-MONEY        189\n",
       "B-DATE         168\n",
       "L-DATE         168\n",
       "U-PERCENT      160\n",
       "I-MONEY        154\n",
       "I-PERS         151\n",
       "I-LOC          149\n",
       "B-MISC         109\n",
       "L-MISC         109\n",
       "I-DATE          98\n",
       "I-MISC          95\n",
       "B-PERCENT       47\n",
       "L-PERCENT       47\n",
       "U-MONEY         19\n",
       "L-TIME          18\n",
       "B-TIME          18\n",
       "I-TIME          12\n",
       "U-TIME          10\n",
       "I-PERCENT        3\n",
       "Name: BILUO, dtype: int64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['BILUO'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "spec_class_data = df[df['BILUO'] == 'L-MISC']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word</th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Token</th>\n",
       "      <th>Prefix</th>\n",
       "      <th>Suffix</th>\n",
       "      <th>Bio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1031</th>\n",
       "      <td>הרומאים</td>\n",
       "      <td>רומאי</td>\n",
       "      <td>והרומאים</td>\n",
       "      <td>וה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>B_MISC__AFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1935</th>\n",
       "      <td>האדם</td>\n",
       "      <td>אדם</td>\n",
       "      <td>האדם</td>\n",
       "      <td>ה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2108</th>\n",
       "      <td>מש'</td>\n",
       "      <td>מש'</td>\n",
       "      <td>ממש'</td>\n",
       "      <td>מ</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3157</th>\n",
       "      <td>יפניים</td>\n",
       "      <td>יפני</td>\n",
       "      <td>ליפניים</td>\n",
       "      <td>ל</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>B_MISC__AFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3717</th>\n",
       "      <td>וולקן</td>\n",
       "      <td>וולקן</td>\n",
       "      <td>וולקן</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4490</th>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4511</th>\n",
       "      <td>שדים</td>\n",
       "      <td>שד</td>\n",
       "      <td>ושדים</td>\n",
       "      <td>ו</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4545</th>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4585</th>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4805</th>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4856</th>\n",
       "      <td>החדשה</td>\n",
       "      <td>חדשה</td>\n",
       "      <td>החדשה</td>\n",
       "      <td>ה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4941</th>\n",
       "      <td>ג'מיני</td>\n",
       "      <td>ג'מיני</td>\n",
       "      <td>ג'מיני</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4953</th>\n",
       "      <td>הצלב</td>\n",
       "      <td>צלב</td>\n",
       "      <td>הצלב</td>\n",
       "      <td>ה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4985</th>\n",
       "      <td>החדשה</td>\n",
       "      <td>חדשה</td>\n",
       "      <td>החדשה</td>\n",
       "      <td>ה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5194</th>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5871</th>\n",
       "      <td>הקדוש</td>\n",
       "      <td>קדוש</td>\n",
       "      <td>הקדוש</td>\n",
       "      <td>ה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6124</th>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>וינצ'י</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6229</th>\n",
       "      <td>שלמה</td>\n",
       "      <td>שלם</td>\n",
       "      <td>שלמה</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9339</th>\n",
       "      <td>יהודים</td>\n",
       "      <td>יהודי</td>\n",
       "      <td>ויהודים</td>\n",
       "      <td>ו</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>B_MISC__AFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9551</th>\n",
       "      <td>אמריקאים</td>\n",
       "      <td>אמריקני</td>\n",
       "      <td>אמריקאים</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>B_MISC__AFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10762</th>\n",
       "      <td>איכות</td>\n",
       "      <td>איכות</td>\n",
       "      <td>איכות</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10774</th>\n",
       "      <td>מאושרת</td>\n",
       "      <td>מאושר</td>\n",
       "      <td>מאושרת</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11176</th>\n",
       "      <td>הנביאים</td>\n",
       "      <td>נביא</td>\n",
       "      <td>הנביאים</td>\n",
       "      <td>ה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11191</th>\n",
       "      <td>נבחרות</td>\n",
       "      <td>נבחר</td>\n",
       "      <td>נבחרות</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11769</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11799</th>\n",
       "      <td>זורחת</td>\n",
       "      <td>זורח</td>\n",
       "      <td>זורחת</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12062</th>\n",
       "      <td>אביבי</td>\n",
       "      <td>אביב</td>\n",
       "      <td>אביבי</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__AFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13281</th>\n",
       "      <td>בית</td>\n",
       "      <td>בית</td>\n",
       "      <td>מבית</td>\n",
       "      <td>מ</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13483</th>\n",
       "      <td>האמריקאי</td>\n",
       "      <td>אמריקני</td>\n",
       "      <td>האמריקאי</td>\n",
       "      <td>ה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13834</th>\n",
       "      <td>ריווי</td>\n",
       "      <td>ריווי</td>\n",
       "      <td>ריוויו</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>ו</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38371</th>\n",
       "      <td>השנייה</td>\n",
       "      <td>ORD2</td>\n",
       "      <td>השנייה</td>\n",
       "      <td>ה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38989</th>\n",
       "      <td>זנווה</td>\n",
       "      <td>ז'נבה</td>\n",
       "      <td>זנווה</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39046</th>\n",
       "      <td>אמריקאי</td>\n",
       "      <td>אמריקני</td>\n",
       "      <td>אמריקאי</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>B_MISC__AFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39169</th>\n",
       "      <td>האלופות</td>\n",
       "      <td>אלופה</td>\n",
       "      <td>האלופות</td>\n",
       "      <td>ה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39218</th>\n",
       "      <td>האלופות</td>\n",
       "      <td>אלופה</td>\n",
       "      <td>האלופות</td>\n",
       "      <td>ה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39242</th>\n",
       "      <td>האירופית</td>\n",
       "      <td>אירופי</td>\n",
       "      <td>האירופית</td>\n",
       "      <td>ה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39257</th>\n",
       "      <td>משקולות</td>\n",
       "      <td>משקולת</td>\n",
       "      <td>משקולות</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39632</th>\n",
       "      <td>מונטריאול</td>\n",
       "      <td>מונטריאול</td>\n",
       "      <td>מונטריאול</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40206</th>\n",
       "      <td>וייטנאם</td>\n",
       "      <td>וייטנאם</td>\n",
       "      <td>וייטנאם</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40428</th>\n",
       "      <td>הקרה</td>\n",
       "      <td>קרה</td>\n",
       "      <td>הקרה</td>\n",
       "      <td>ה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41502</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42485</th>\n",
       "      <td>בשבע'</td>\n",
       "      <td>בשבע'</td>\n",
       "      <td>בשבע'</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42609</th>\n",
       "      <td>ג'נבה</td>\n",
       "      <td>ז'נבה</td>\n",
       "      <td>וג'נבה</td>\n",
       "      <td>ו</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>B_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45654</th>\n",
       "      <td>הימים</td>\n",
       "      <td>יום</td>\n",
       "      <td>הימים</td>\n",
       "      <td>ה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46227</th>\n",
       "      <td>אוסלו</td>\n",
       "      <td>אוסלו</td>\n",
       "      <td>אוסלו</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47132</th>\n",
       "      <td>ישראלי</td>\n",
       "      <td>ישראלי</td>\n",
       "      <td>ישראלי</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>B_MISC__AFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47676</th>\n",
       "      <td>יאטנם</td>\n",
       "      <td>יאטנם</td>\n",
       "      <td>ויאטנם</td>\n",
       "      <td>ו</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47836</th>\n",
       "      <td>ישראלים</td>\n",
       "      <td>ישראלי</td>\n",
       "      <td>ישראלים</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>B_MISC__AFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48510</th>\n",
       "      <td>אוסלו</td>\n",
       "      <td>אוסלו</td>\n",
       "      <td>אוסלו</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50029</th>\n",
       "      <td>אירופה</td>\n",
       "      <td>אירופה</td>\n",
       "      <td>אירופה</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50173</th>\n",
       "      <td>מינכן</td>\n",
       "      <td>מינכן</td>\n",
       "      <td>מינכן</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50812</th>\n",
       "      <td>מוט</td>\n",
       "      <td>מוט</td>\n",
       "      <td>במוט</td>\n",
       "      <td>ב</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51156</th>\n",
       "      <td>הפלשתינית</td>\n",
       "      <td>פלסטיני</td>\n",
       "      <td>הפלשתינית</td>\n",
       "      <td>ה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__AFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51413</th>\n",
       "      <td>ירדנים</td>\n",
       "      <td>ירדני</td>\n",
       "      <td>וירדנים</td>\n",
       "      <td>ו</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>B_MISC__AFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52069</th>\n",
       "      <td>ירושלים</td>\n",
       "      <td>ירושלים</td>\n",
       "      <td>לירושלים</td>\n",
       "      <td>ל</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52133</th>\n",
       "      <td>ירושלים</td>\n",
       "      <td>ירושלים</td>\n",
       "      <td>לירושלים</td>\n",
       "      <td>ל</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52179</th>\n",
       "      <td>חרדיים</td>\n",
       "      <td>חרדי</td>\n",
       "      <td>חרדיים</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>B_MISC__AFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52237</th>\n",
       "      <td>סכה</td>\n",
       "      <td>סך</td>\n",
       "      <td>מסכה</td>\n",
       "      <td>מ</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__ENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52555</th>\n",
       "      <td>פלשתינים</td>\n",
       "      <td>פלסטיני</td>\n",
       "      <td>ופלשתינים</td>\n",
       "      <td>ו</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>B_MISC__AFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53282</th>\n",
       "      <td>הימים</td>\n",
       "      <td>יום</td>\n",
       "      <td>הימים</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>109 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               Word         Lemma         Token   Prefix     Suffix  \\\n",
       "1031        הרומאים         רומאי      והרומאים       וה  no_suffix   \n",
       "1935           האדם           אדם          האדם        ה  no_suffix   \n",
       "2108            מש'           מש'          ממש'        מ  no_suffix   \n",
       "3157         יפניים          יפני       ליפניים        ל  no_suffix   \n",
       "3717          וולקן         וולקן         וולקן  no_pref  no_suffix   \n",
       "4490         וינצ'י        וינצ'י        וינצ'י  no_pref  no_suffix   \n",
       "4511           שדים            שד         ושדים        ו  no_suffix   \n",
       "4545         וינצ'י        וינצ'י        וינצ'י  no_pref  no_suffix   \n",
       "4585         וינצ'י        וינצ'י        וינצ'י  no_pref  no_suffix   \n",
       "4805         וינצ'י        וינצ'י        וינצ'י  no_pref  no_suffix   \n",
       "4856          החדשה          חדשה         החדשה        ה  no_suffix   \n",
       "4941         ג'מיני        ג'מיני        ג'מיני  no_pref  no_suffix   \n",
       "4953           הצלב           צלב          הצלב        ה  no_suffix   \n",
       "4985          החדשה          חדשה         החדשה        ה  no_suffix   \n",
       "5194         וינצ'י        וינצ'י        וינצ'י  no_pref  no_suffix   \n",
       "5871          הקדוש          קדוש         הקדוש        ה  no_suffix   \n",
       "6124         וינצ'י        וינצ'י        וינצ'י  no_pref  no_suffix   \n",
       "6229           שלמה           שלם          שלמה  no_pref  no_suffix   \n",
       "9339         יהודים         יהודי       ויהודים        ו  no_suffix   \n",
       "9551       אמריקאים       אמריקני      אמריקאים  no_pref  no_suffix   \n",
       "10762         איכות         איכות         איכות  no_pref  no_suffix   \n",
       "10774        מאושרת         מאושר        מאושרת  no_pref  no_suffix   \n",
       "11176       הנביאים          נביא       הנביאים        ה  no_suffix   \n",
       "11191        נבחרות          נבחר        נבחרות  no_pref  no_suffix   \n",
       "11769  ###NUMBER###  ###NUMBER###  ###NUMBER###  no_pref  no_suffix   \n",
       "11799         זורחת          זורח         זורחת  no_pref  no_suffix   \n",
       "12062         אביבי          אביב         אביבי  no_pref  no_suffix   \n",
       "13281           בית           בית          מבית        מ  no_suffix   \n",
       "13483      האמריקאי       אמריקני      האמריקאי        ה  no_suffix   \n",
       "13834         ריווי         ריווי        ריוויו  no_pref          ו   \n",
       "...             ...           ...           ...      ...        ...   \n",
       "38371        השנייה          ORD2        השנייה        ה  no_suffix   \n",
       "38989         זנווה         ז'נבה         זנווה  no_pref  no_suffix   \n",
       "39046       אמריקאי       אמריקני       אמריקאי  no_pref  no_suffix   \n",
       "39169       האלופות         אלופה       האלופות        ה  no_suffix   \n",
       "39218       האלופות         אלופה       האלופות        ה  no_suffix   \n",
       "39242      האירופית        אירופי      האירופית        ה  no_suffix   \n",
       "39257       משקולות        משקולת       משקולות  no_pref  no_suffix   \n",
       "39632     מונטריאול     מונטריאול     מונטריאול  no_pref  no_suffix   \n",
       "40206       וייטנאם       וייטנאם       וייטנאם  no_pref  no_suffix   \n",
       "40428          הקרה           קרה          הקרה        ה  no_suffix   \n",
       "41502  ###NUMBER###  ###NUMBER###  ###NUMBER###  no_pref  no_suffix   \n",
       "42485         בשבע'         בשבע'         בשבע'  no_pref  no_suffix   \n",
       "42609         ג'נבה         ז'נבה        וג'נבה        ו  no_suffix   \n",
       "45654         הימים           יום         הימים        ה  no_suffix   \n",
       "46227         אוסלו         אוסלו         אוסלו  no_pref  no_suffix   \n",
       "47132        ישראלי        ישראלי        ישראלי  no_pref  no_suffix   \n",
       "47676         יאטנם         יאטנם        ויאטנם        ו  no_suffix   \n",
       "47836       ישראלים        ישראלי       ישראלים  no_pref  no_suffix   \n",
       "48510         אוסלו         אוסלו         אוסלו  no_pref  no_suffix   \n",
       "50029        אירופה        אירופה        אירופה  no_pref  no_suffix   \n",
       "50173         מינכן         מינכן         מינכן  no_pref  no_suffix   \n",
       "50812           מוט           מוט          במוט        ב  no_suffix   \n",
       "51156     הפלשתינית       פלסטיני     הפלשתינית        ה  no_suffix   \n",
       "51413        ירדנים         ירדני       וירדנים        ו  no_suffix   \n",
       "52069       ירושלים       ירושלים      לירושלים        ל  no_suffix   \n",
       "52133       ירושלים       ירושלים      לירושלים        ל  no_suffix   \n",
       "52179        חרדיים          חרדי        חרדיים  no_pref  no_suffix   \n",
       "52237           סכה            סך          מסכה        מ  no_suffix   \n",
       "52555      פלשתינים       פלסטיני     ופלשתינים        ו  no_suffix   \n",
       "53282         הימים           יום         הימים  no_pref  no_suffix   \n",
       "\n",
       "                Bio  \n",
       "1031    B_MISC__AFF  \n",
       "1935   I_MISC_EVENT  \n",
       "2108    I_MISC__ENT  \n",
       "3157    B_MISC__AFF  \n",
       "3717    I_MISC__ENT  \n",
       "4490    I_MISC__ENT  \n",
       "4511    I_MISC__ENT  \n",
       "4545    I_MISC__ENT  \n",
       "4585    I_MISC__ENT  \n",
       "4805    I_MISC__ENT  \n",
       "4856    I_MISC__ENT  \n",
       "4941    I_MISC__ENT  \n",
       "4953    I_MISC__ENT  \n",
       "4985    I_MISC__ENT  \n",
       "5194    I_MISC__ENT  \n",
       "5871    I_MISC__ENT  \n",
       "6124    I_MISC__ENT  \n",
       "6229    I_MISC__ENT  \n",
       "9339    B_MISC__AFF  \n",
       "9551    B_MISC__AFF  \n",
       "10762  I_MISC_EVENT  \n",
       "10774   I_MISC__ENT  \n",
       "11176   I_MISC__ENT  \n",
       "11191   I_MISC__ENT  \n",
       "11769   I_MISC__ENT  \n",
       "11799   I_MISC__ENT  \n",
       "12062   I_MISC__AFF  \n",
       "13281   I_MISC__ENT  \n",
       "13483   I_MISC__ENT  \n",
       "13834   I_MISC__ENT  \n",
       "...             ...  \n",
       "38371  I_MISC_EVENT  \n",
       "38989   I_MISC__ENT  \n",
       "39046   B_MISC__AFF  \n",
       "39169  I_MISC_EVENT  \n",
       "39218  I_MISC_EVENT  \n",
       "39242  I_MISC_EVENT  \n",
       "39257  I_MISC_EVENT  \n",
       "39632  I_MISC_EVENT  \n",
       "40206  I_MISC_EVENT  \n",
       "40428  I_MISC_EVENT  \n",
       "41502   I_MISC__ENT  \n",
       "42485   I_MISC__ENT  \n",
       "42609   B_MISC__ENT  \n",
       "45654  I_MISC_EVENT  \n",
       "46227   I_MISC__ENT  \n",
       "47132   B_MISC__AFF  \n",
       "47676  I_MISC_EVENT  \n",
       "47836   B_MISC__AFF  \n",
       "48510  I_MISC_EVENT  \n",
       "50029  I_MISC_EVENT  \n",
       "50173  I_MISC_EVENT  \n",
       "50812  I_MISC_EVENT  \n",
       "51156   I_MISC__AFF  \n",
       "51413   B_MISC__AFF  \n",
       "52069  I_MISC_EVENT  \n",
       "52133  I_MISC_EVENT  \n",
       "52179   B_MISC__AFF  \n",
       "52237   I_MISC__ENT  \n",
       "52555   B_MISC__AFF  \n",
       "53282  I_MISC_EVENT  \n",
       "\n",
       "[109 rows x 6 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spec_class_data[['Word','Lemma','Token','Prefix','Suffix','Bio']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_spec_class_data = df[df['BILUO'] != 'L-MISC']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "unspecified    11\n",
       "infinitive      1\n",
       "Name: Tense, dtype: int64"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spec_class_data['Tense'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "unspecified    154\n",
       "absolute        96\n",
       "construct       17\n",
       "Name: Status, dtype: int64"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spec_class_data['Status'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "unspecified    36148\n",
       "absolute       23510\n",
       "construct       2986\n",
       "Name: Status, dtype: int64"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not_spec_class_data['Status'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "unspecified    264\n",
       "1                1\n",
       "any              1\n",
       "3                1\n",
       "Name: Person, dtype: int64"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spec_class_data['Person'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "unspecified    53830\n",
       "3               5440\n",
       "any             2266\n",
       "1                566\n",
       "2                542\n",
       "Name: Person, dtype: int64"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not_spec_class_data['Person'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word</th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Token</th>\n",
       "      <th>Prefix</th>\n",
       "      <th>Suffix</th>\n",
       "      <th>Bio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>אחד</td>\n",
       "      <td>CARD1</td>\n",
       "      <td>אחד</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>כל</td>\n",
       "      <td>כול</td>\n",
       "      <td>מכל</td>\n",
       "      <td>מ</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>שני</td>\n",
       "      <td>שני</td>\n",
       "      <td>שני</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ישראלים</td>\n",
       "      <td>ישראלי</td>\n",
       "      <td>ישראלים</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MISC__AFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>אוכלים</td>\n",
       "      <td>אוכל</td>\n",
       "      <td>אוכלים</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>שליש</td>\n",
       "      <td>שליש</td>\n",
       "      <td>שליש</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>הם</td>\n",
       "      <td>הם</td>\n",
       "      <td>מהם</td>\n",
       "      <td>מן</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>נהנים</td>\n",
       "      <td>נהנה</td>\n",
       "      <td>נהנים</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>נו</td>\n",
       "      <td>נו</td>\n",
       "      <td>ממנו</td>\n",
       "      <td>מן</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>בוקר</td>\n",
       "      <td>בוקר</td>\n",
       "      <td>בבוקר</td>\n",
       "      <td>ב</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_PERCENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>אחוז</td>\n",
       "      <td>אחוז</td>\n",
       "      <td>אחוז</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_PERCENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>אוכלים</td>\n",
       "      <td>אוכל</td>\n",
       "      <td>אוכלים</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>אותו</td>\n",
       "      <td>אותו</td>\n",
       "      <td>אותו</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ערב</td>\n",
       "      <td>ערב</td>\n",
       "      <td>בערב</td>\n",
       "      <td>ב</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>אילו</td>\n",
       "      <td>אילו</td>\n",
       "      <td>ואילו</td>\n",
       "      <td>ו</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_PERCENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>%</td>\n",
       "      <td>%</td>\n",
       "      <td>%</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_PERCENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>אוכלים</td>\n",
       "      <td>אוכל</td>\n",
       "      <td>אוכלים</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>צהריים</td>\n",
       "      <td>צהריים</td>\n",
       "      <td>בצהריים</td>\n",
       "      <td>ב</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>נתון</td>\n",
       "      <td>נתן</td>\n",
       "      <td>נתון</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>עניין</td>\n",
       "      <td>עניין</td>\n",
       "      <td>מעניין</td>\n",
       "      <td>מ</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>נוסף</td>\n",
       "      <td>נוסף</td>\n",
       "      <td>נוסף</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62881</th>\n",
       "      <td>דולר</td>\n",
       "      <td>דולר</td>\n",
       "      <td>דולר</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MONEY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62882</th>\n",
       "      <td>רווח</td>\n",
       "      <td>רווח</td>\n",
       "      <td>ורווח</td>\n",
       "      <td>ו</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62883</th>\n",
       "      <td>נקי</td>\n",
       "      <td>נקי</td>\n",
       "      <td>נקי</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62884</th>\n",
       "      <td>של</td>\n",
       "      <td>של</td>\n",
       "      <td>של</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62885</th>\n",
       "      <td>לפחות</td>\n",
       "      <td>לפחות</td>\n",
       "      <td>לפחות</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62886</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MONEY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62887</th>\n",
       "      <td>מיליון</td>\n",
       "      <td>ORD-CARD1000000</td>\n",
       "      <td>מיליון</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MONEY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62888</th>\n",
       "      <td>דולר</td>\n",
       "      <td>דולר</td>\n",
       "      <td>דולר</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_MONEY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62889</th>\n",
       "      <td>שנת</td>\n",
       "      <td>שינה</td>\n",
       "      <td>בשנת</td>\n",
       "      <td>ב</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62890</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_DATE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62891</th>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62892</th>\n",
       "      <td>ריטליקס</td>\n",
       "      <td>ריטליקס</td>\n",
       "      <td>ריטליקס</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>I_ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62893</th>\n",
       "      <td>מפתחת</td>\n",
       "      <td>פיתח</td>\n",
       "      <td>מפתחת</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62894</th>\n",
       "      <td>משווקת</td>\n",
       "      <td>שיווק</td>\n",
       "      <td>ומשווקת</td>\n",
       "      <td>ו</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62895</th>\n",
       "      <td>פתרונות</td>\n",
       "      <td>פתרון</td>\n",
       "      <td>פתרונות</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62896</th>\n",
       "      <td>תוכנה</td>\n",
       "      <td>תיווך</td>\n",
       "      <td>תוכנה</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62897</th>\n",
       "      <td>כוללים</td>\n",
       "      <td>כולל</td>\n",
       "      <td>כוללים</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62898</th>\n",
       "      <td>תעשיית</td>\n",
       "      <td>תעשייה</td>\n",
       "      <td>לתעשיית</td>\n",
       "      <td>ל</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62899</th>\n",
       "      <td>קמעונאות</td>\n",
       "      <td>קמעונאות</td>\n",
       "      <td>קמעונאות</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62900</th>\n",
       "      <td>המזון</td>\n",
       "      <td>מזון</td>\n",
       "      <td>המזון</td>\n",
       "      <td>ה</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62901</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62902</th>\n",
       "      <td>סופרמרקטים</td>\n",
       "      <td>סופרמרקט</td>\n",
       "      <td>סופרמרקטים</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62903</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62904</th>\n",
       "      <td>חנויות</td>\n",
       "      <td>חנות</td>\n",
       "      <td>חנויות</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62905</th>\n",
       "      <td>נוחות</td>\n",
       "      <td>נוח</td>\n",
       "      <td>נוחות</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62906</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62907</th>\n",
       "      <td>תחנות</td>\n",
       "      <td>תחנה</td>\n",
       "      <td>תחנות</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62908</th>\n",
       "      <td>דלק</td>\n",
       "      <td>דלק</td>\n",
       "      <td>דלק</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62909</th>\n",
       "      <td>מסעדות</td>\n",
       "      <td>מסעדה</td>\n",
       "      <td>ומסעדות</td>\n",
       "      <td>ו</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62910</th>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>no_pref</td>\n",
       "      <td>no_suffix</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>62911 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               Word            Lemma         Token   Prefix     Suffix  \\\n",
       "0          DOCSTART         DOCSTART      DOCSTART  no_pref  no_suffix   \n",
       "1               אחד            CARD1           אחד  no_pref  no_suffix   \n",
       "2                כל              כול           מכל        מ  no_suffix   \n",
       "3               שני              שני           שני  no_pref  no_suffix   \n",
       "4           ישראלים           ישראלי       ישראלים  no_pref  no_suffix   \n",
       "5            אוכלים             אוכל        אוכלים  no_pref  no_suffix   \n",
       "6          קורנפלקס         קורנפלקס      קורנפלקס  no_pref  no_suffix   \n",
       "7                 .                .             .  no_pref  no_suffix   \n",
       "8              שליש             שליש          שליש  no_pref  no_suffix   \n",
       "9                הם               הם           מהם       מן  no_suffix   \n",
       "10            נהנים             נהנה         נהנים  no_pref  no_suffix   \n",
       "11               נו               נו          ממנו       מן  no_suffix   \n",
       "12             בוקר             בוקר         בבוקר        ב  no_suffix   \n",
       "13                ,                ,             ,  no_pref  no_suffix   \n",
       "14     ###NUMBER###     ###NUMBER###  ###NUMBER###  no_pref  no_suffix   \n",
       "15             אחוז             אחוז          אחוז  no_pref  no_suffix   \n",
       "16           אוכלים             אוכל        אוכלים  no_pref  no_suffix   \n",
       "17             אותו             אותו          אותו  no_pref  no_suffix   \n",
       "18              ערב              ערב          בערב        ב  no_suffix   \n",
       "19                ,                ,             ,  no_pref  no_suffix   \n",
       "20             אילו             אילו         ואילו        ו  no_suffix   \n",
       "21     ###NUMBER###     ###NUMBER###  ###NUMBER###  no_pref  no_suffix   \n",
       "22                %                %             %  no_pref  no_suffix   \n",
       "23           אוכלים             אוכל        אוכלים  no_pref  no_suffix   \n",
       "24         קורנפלקס         קורנפלקס      קורנפלקס  no_pref  no_suffix   \n",
       "25           צהריים           צהריים       בצהריים        ב  no_suffix   \n",
       "26                .                .             .  no_pref  no_suffix   \n",
       "27             נתון              נתן          נתון  no_pref  no_suffix   \n",
       "28            עניין            עניין        מעניין        מ  no_suffix   \n",
       "29             נוסף             נוסף          נוסף  no_pref  no_suffix   \n",
       "...             ...              ...           ...      ...        ...   \n",
       "62881          דולר             דולר          דולר  no_pref  no_suffix   \n",
       "62882          רווח             רווח         ורווח        ו  no_suffix   \n",
       "62883           נקי              נקי           נקי  no_pref  no_suffix   \n",
       "62884            של               של            של  no_pref  no_suffix   \n",
       "62885         לפחות            לפחות         לפחות  no_pref  no_suffix   \n",
       "62886  ###NUMBER###     ###NUMBER###  ###NUMBER###  no_pref  no_suffix   \n",
       "62887        מיליון  ORD-CARD1000000        מיליון  no_pref  no_suffix   \n",
       "62888          דולר             דולר          דולר  no_pref  no_suffix   \n",
       "62889           שנת             שינה          בשנת        ב  no_suffix   \n",
       "62890  ###NUMBER###     ###NUMBER###  ###NUMBER###  no_pref  no_suffix   \n",
       "62891             .                .             .  no_pref  no_suffix   \n",
       "62892       ריטליקס          ריטליקס       ריטליקס  no_pref  no_suffix   \n",
       "62893         מפתחת             פיתח         מפתחת  no_pref  no_suffix   \n",
       "62894        משווקת            שיווק       ומשווקת        ו  no_suffix   \n",
       "62895       פתרונות            פתרון       פתרונות  no_pref  no_suffix   \n",
       "62896         תוכנה            תיווך         תוכנה  no_pref  no_suffix   \n",
       "62897        כוללים             כולל        כוללים  no_pref  no_suffix   \n",
       "62898        תעשיית           תעשייה       לתעשיית        ל  no_suffix   \n",
       "62899      קמעונאות         קמעונאות      קמעונאות  no_pref  no_suffix   \n",
       "62900         המזון             מזון         המזון        ה  no_suffix   \n",
       "62901             ,                ,             ,  no_pref  no_suffix   \n",
       "62902    סופרמרקטים         סופרמרקט    סופרמרקטים  no_pref  no_suffix   \n",
       "62903             ,                ,             ,  no_pref  no_suffix   \n",
       "62904        חנויות             חנות        חנויות  no_pref  no_suffix   \n",
       "62905         נוחות              נוח         נוחות  no_pref  no_suffix   \n",
       "62906             ,                ,             ,  no_pref  no_suffix   \n",
       "62907         תחנות             תחנה         תחנות  no_pref  no_suffix   \n",
       "62908           דלק              דלק           דלק  no_pref  no_suffix   \n",
       "62909        מסעדות            מסעדה       ומסעדות        ו  no_suffix   \n",
       "62910             .                .             .  no_pref  no_suffix   \n",
       "\n",
       "               Bio  \n",
       "0                O  \n",
       "1                O  \n",
       "2                O  \n",
       "3                O  \n",
       "4      I_MISC__AFF  \n",
       "5                O  \n",
       "6                O  \n",
       "7                O  \n",
       "8                O  \n",
       "9                O  \n",
       "10               O  \n",
       "11               O  \n",
       "12               O  \n",
       "13               O  \n",
       "14       I_PERCENT  \n",
       "15       I_PERCENT  \n",
       "16               O  \n",
       "17               O  \n",
       "18               O  \n",
       "19               O  \n",
       "20               O  \n",
       "21       I_PERCENT  \n",
       "22       I_PERCENT  \n",
       "23               O  \n",
       "24               O  \n",
       "25               O  \n",
       "26               O  \n",
       "27               O  \n",
       "28               O  \n",
       "29               O  \n",
       "...            ...  \n",
       "62881      I_MONEY  \n",
       "62882            O  \n",
       "62883            O  \n",
       "62884            O  \n",
       "62885            O  \n",
       "62886      I_MONEY  \n",
       "62887      I_MONEY  \n",
       "62888      I_MONEY  \n",
       "62889            O  \n",
       "62890       I_DATE  \n",
       "62891            O  \n",
       "62892        I_ORG  \n",
       "62893            O  \n",
       "62894            O  \n",
       "62895            O  \n",
       "62896            O  \n",
       "62897            O  \n",
       "62898            O  \n",
       "62899            O  \n",
       "62900            O  \n",
       "62901            O  \n",
       "62902            O  \n",
       "62903            O  \n",
       "62904            O  \n",
       "62905            O  \n",
       "62906            O  \n",
       "62907            O  \n",
       "62908            O  \n",
       "62909            O  \n",
       "62910            O  \n",
       "\n",
       "[62911 rows x 6 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['Word','Lemma','Token','Prefix','Suffix','Bio']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'resources' + os.sep + 'dataset.csv'    \n",
    "df = pd.read_csv(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df = df[df['Word'].apply(lambda x: len(x) > 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bio</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Number</th>\n",
       "      <th>Person</th>\n",
       "      <th>Pos</th>\n",
       "      <th>Status</th>\n",
       "      <th>Tense</th>\n",
       "      <th>Token</th>\n",
       "      <th>TokenOrder</th>\n",
       "      <th>Word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>foreign</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\n</td>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>99</td>\n",
       "      <td>DOCSTART</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>O</td>\n",
       "      <td>masculine</td>\n",
       "      <td>CARD1</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>numeral</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\n</td>\n",
       "      <td>אחד</td>\n",
       "      <td>102</td>\n",
       "      <td>אחד</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>O</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>כול</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>quantifier</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\n</td>\n",
       "      <td>מכל</td>\n",
       "      <td>103</td>\n",
       "      <td>כל</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>O</td>\n",
       "      <td>masculine</td>\n",
       "      <td>שני</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\n</td>\n",
       "      <td>שני</td>\n",
       "      <td>104</td>\n",
       "      <td>שני</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>I_MISC__AFF</td>\n",
       "      <td>masculine</td>\n",
       "      <td>ישראלי</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\n</td>\n",
       "      <td>ישראלים</td>\n",
       "      <td>105</td>\n",
       "      <td>ישראלים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>O</td>\n",
       "      <td>masculine</td>\n",
       "      <td>אוכל</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\n</td>\n",
       "      <td>אוכלים</td>\n",
       "      <td>106</td>\n",
       "      <td>אוכלים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>O</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\n</td>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>107</td>\n",
       "      <td>קורנפלקס</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>O</td>\n",
       "      <td>masculine</td>\n",
       "      <td>שליש</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>numeral</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\n</td>\n",
       "      <td>שליש</td>\n",
       "      <td>109</td>\n",
       "      <td>שליש</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>O</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>מן</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>preposition</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\n</td>\n",
       "      <td>מהם</td>\n",
       "      <td>110</td>\n",
       "      <td>מן</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>O</td>\n",
       "      <td>masculine</td>\n",
       "      <td>הם</td>\n",
       "      <td>plural</td>\n",
       "      <td>3</td>\n",
       "      <td>pronomial</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\n</td>\n",
       "      <td>מהם</td>\n",
       "      <td>110</td>\n",
       "      <td>הם</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Bio       Gender     Lemma       Number       Person          Pos  \\\n",
       "0             O  unspecified  DOCSTART  unspecified  unspecified      foreign   \n",
       "1             O    masculine     CARD1     singular  unspecified      numeral   \n",
       "3             O  unspecified       כול  unspecified  unspecified   quantifier   \n",
       "4             O    masculine       שני     singular  unspecified         noun   \n",
       "5   I_MISC__AFF    masculine    ישראלי       plural  unspecified         noun   \n",
       "6             O    masculine      אוכל       plural  unspecified         noun   \n",
       "7             O  unspecified  קורנפלקס  unspecified  unspecified   properName   \n",
       "9             O    masculine      שליש     singular  unspecified      numeral   \n",
       "10            O  unspecified        מן  unspecified  unspecified  preposition   \n",
       "11            O    masculine        הם       plural            3    pronomial   \n",
       "\n",
       "         Status            Tense     Token  TokenOrder      Word  \n",
       "0   unspecified  unspecified\\r\\n  DOCSTART          99  DOCSTART  \n",
       "1      absolute  unspecified\\r\\n       אחד         102       אחד  \n",
       "3     construct  unspecified\\r\\n       מכל         103        כל  \n",
       "4      absolute  unspecified\\r\\n       שני         104       שני  \n",
       "5      absolute  unspecified\\r\\n   ישראלים         105   ישראלים  \n",
       "6      absolute  unspecified\\r\\n    אוכלים         106    אוכלים  \n",
       "7   unspecified  unspecified\\r\\n  קורנפלקס         107  קורנפלקס  \n",
       "9      absolute  unspecified\\r\\n      שליש         109      שליש  \n",
       "10  unspecified  unspecified\\r\\n       מהם         110        מן  \n",
       "11  unspecified  unspecified\\r\\n       מהם         110        הם  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "naama_data = 'resources' + os.sep + 'old_files' + os.sep + 'tagged_corpus_naama_trimmed.txt'\n",
    "with open(naama_data, 'r', encoding='utf-8') as f:\n",
    "    naamaz = f.readlines()\n",
    "    \n",
    "naama_data = 'resources' + os.sep + 'old_files' + os.sep + 'tagged_corpus_naama.txt'\n",
    "with open(naama_data, 'r', encoding='utf-8') as f:\n",
    "    naamaz2 = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(66392, 66574)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(naamaz), len(naamaz2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Merged output (from assaf):\n",
    "62911"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3473"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([w for w in naamaz if w == '\\n'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w for w in naamaz if len(w) == 1 and w != '\\n']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "52819"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([w.split(\" \")[0] for w in naamaz if len(w.split(\" \")[0])>1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '3',\n",
       " '%',\n",
       " '.',\n",
       " '\\n',\n",
       " ':',\n",
       " '.',\n",
       " '\\n',\n",
       " '%',\n",
       " ',',\n",
       " '%',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '-',\n",
       " ';',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '6',\n",
       " '%',\n",
       " ',',\n",
       " '1',\n",
       " '%',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '\\n',\n",
       " '-',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '7',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " 'ל',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '%',\n",
       " '%',\n",
       " '.',\n",
       " '\\n',\n",
       " '-',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '\\n',\n",
       " \"'\",\n",
       " \"'\",\n",
       " ',',\n",
       " \"'\",\n",
       " ',',\n",
       " \"'\",\n",
       " 'ר',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " \"'\",\n",
       " 'ר',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " \"'\",\n",
       " \"'\",\n",
       " ',',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " \"'\",\n",
       " 'כ',\n",
       " '.',\n",
       " '\\n',\n",
       " \"'\",\n",
       " \"'\",\n",
       " '.',\n",
       " '\\n',\n",
       " \"'\",\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ':',\n",
       " '\"',\n",
       " \"'\",\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '-',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '?',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '(',\n",
       " '-',\n",
       " ',',\n",
       " ',',\n",
       " ')',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '-',\n",
       " '-',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " 'ה',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '-',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '(',\n",
       " ')',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '-',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '\\n',\n",
       " '4',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ':',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '-',\n",
       " '?',\n",
       " '\\n',\n",
       " '\"',\n",
       " \"'\",\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '(',\n",
       " ')',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ':',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ':',\n",
       " '\"',\n",
       " '\"',\n",
       " '\"',\n",
       " '(',\n",
       " ',',\n",
       " ',',\n",
       " ')',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ':',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '(',\n",
       " ')',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ':',\n",
       " '\"',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ':',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ':',\n",
       " '\"',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ':',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '\\n',\n",
       " '\"',\n",
       " ',',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " 'ה',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '(',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " ')',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '?',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '(',\n",
       " ')',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '(',\n",
       " ';',\n",
       " 'א',\n",
       " '.',\n",
       " 'י',\n",
       " ')',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '\\n',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '(',\n",
       " ')',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " 'ב',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " '(',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " '\"',\n",
       " '\"',\n",
       " ')',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ':',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " ',',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ':',\n",
       " '\"',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '\\n',\n",
       " '2',\n",
       " '.',\n",
       " '\\n',\n",
       " '\\n',\n",
       " '1',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\"',\n",
       " '\\n',\n",
       " ',',\n",
       " '\"',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ':',\n",
       " '-',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " 'ב',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '\\n',\n",
       " ',',\n",
       " '\"',\n",
       " '\"',\n",
       " 'ס',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " ',',\n",
       " '(',\n",
       " ')',\n",
       " ',',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '/',\n",
       " ',',\n",
       " '-',\n",
       " ',',\n",
       " '-',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " ',',\n",
       " 'ד',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " ':',\n",
       " ',',\n",
       " ',',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " '.',\n",
       " '\\n',\n",
       " ',',\n",
       " 'מ',\n",
       " '.',\n",
       " '\\n',\n",
       " '.',\n",
       " '\\n',\n",
       " 'ב',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " '\"',\n",
       " '\"',\n",
       " '(',\n",
       " ')',\n",
       " '(',\n",
       " ')',\n",
       " '\"',\n",
       " '\"',\n",
       " ',',\n",
       " '\"',\n",
       " '\"',\n",
       " '.',\n",
       " '\\n',\n",
       " ':',\n",
       " '.',\n",
       " ...]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[w.split(\" \")[0] for w in naamaz if len(w.split(\" \")[0])==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'resources' + os.sep + 'dataset.csv'    \n",
    "df = pd.read_csv(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df = df[df['Word'].apply(lambda x: len(x) > 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = 400"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word</th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Token</th>\n",
       "      <th>Bio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>648</th>\n",
       "      <td>שישה</td>\n",
       "      <td>CARD6</td>\n",
       "      <td>שישה</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650</th>\n",
       "      <td>שטח</td>\n",
       "      <td>שטח</td>\n",
       "      <td>בשטח</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651</th>\n",
       "      <td>ישראל</td>\n",
       "      <td>ישראל</td>\n",
       "      <td>ישראל</td>\n",
       "      <td>I_LOC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>653</th>\n",
       "      <td>שמונה</td>\n",
       "      <td>CARD8</td>\n",
       "      <td>ושמונה</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>655</th>\n",
       "      <td>שטחים</td>\n",
       "      <td>שטח</td>\n",
       "      <td>בשטחים</td>\n",
       "      <td>I_LOC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>658</th>\n",
       "      <td>דברי</td>\n",
       "      <td>דבר</td>\n",
       "      <td>לדבריו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>661</th>\n",
       "      <td>כוחות</td>\n",
       "      <td>כוח</td>\n",
       "      <td>כוחות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>663</th>\n",
       "      <td>הביטחון</td>\n",
       "      <td>ביטחון</td>\n",
       "      <td>הביטחון</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>664</th>\n",
       "      <td>סיכלו</td>\n",
       "      <td>סיכל</td>\n",
       "      <td>סיכלו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>665</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>666</th>\n",
       "      <td>פיגועי</td>\n",
       "      <td>פיגוע</td>\n",
       "      <td>פיגועי</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>667</th>\n",
       "      <td>התאבדות</td>\n",
       "      <td>התאבדות</td>\n",
       "      <td>התאבדות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>669</th>\n",
       "      <td>תפסו</td>\n",
       "      <td>תפס</td>\n",
       "      <td>ותפסו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>670</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>671</th>\n",
       "      <td>מתאבדים</td>\n",
       "      <td>מתאבד</td>\n",
       "      <td>מתאבדים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>673</th>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>676</th>\n",
       "      <td>יהיה</td>\n",
       "      <td>יש</td>\n",
       "      <td>יהיה</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>678</th>\n",
       "      <td>נו</td>\n",
       "      <td>נו</td>\n",
       "      <td>לנו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>679</th>\n",
       "      <td>מעמד</td>\n",
       "      <td>מעמד</td>\n",
       "      <td>מעמד</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>680</th>\n",
       "      <td>נכבד</td>\n",
       "      <td>נכבד</td>\n",
       "      <td>נכבד</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>682</th>\n",
       "      <td>ממשלה</td>\n",
       "      <td>ממשלה</td>\n",
       "      <td>בממשלה</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>685</th>\n",
       "      <td>היא</td>\n",
       "      <td>הוא</td>\n",
       "      <td>שהיא</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>687</th>\n",
       "      <td>הזירה</td>\n",
       "      <td>זירה</td>\n",
       "      <td>הזירה</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>688</th>\n",
       "      <td>אמיתית</td>\n",
       "      <td>אמיתי</td>\n",
       "      <td>אמיתית</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>690</th>\n",
       "      <td>ביצוע</td>\n",
       "      <td>ביצוע</td>\n",
       "      <td>לביצוע</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>691</th>\n",
       "      <td>שינויים</td>\n",
       "      <td>שינוי</td>\n",
       "      <td>שינויים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>692</th>\n",
       "      <td>משמעותיים</td>\n",
       "      <td>משמעותי</td>\n",
       "      <td>משמעותיים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>694</th>\n",
       "      <td>מישור</td>\n",
       "      <td>מישור</td>\n",
       "      <td>במישור</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>696</th>\n",
       "      <td>המדיני</td>\n",
       "      <td>מדיני</td>\n",
       "      <td>המדיני</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>699</th>\n",
       "      <td>החברתי'</td>\n",
       "      <td>החברתי'</td>\n",
       "      <td>והחברתי'</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>769</th>\n",
       "      <td>בירור</td>\n",
       "      <td>בירור</td>\n",
       "      <td>לבירור</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>770</th>\n",
       "      <td>עתירות</td>\n",
       "      <td>עתירה</td>\n",
       "      <td>עתירות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>771</th>\n",
       "      <td>של</td>\n",
       "      <td>של</td>\n",
       "      <td>של</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>773</th>\n",
       "      <td>המפלגה</td>\n",
       "      <td>מפלגה</td>\n",
       "      <td>המפלגה</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>774</th>\n",
       "      <td>את</td>\n",
       "      <td>את</td>\n",
       "      <td>את</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>775</th>\n",
       "      <td>החלטת</td>\n",
       "      <td>החלטה</td>\n",
       "      <td>החלטתו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>777</th>\n",
       "      <td>מאמש</td>\n",
       "      <td>מאמש</td>\n",
       "      <td>מאמש</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>779</th>\n",
       "      <td>על</td>\n",
       "      <td>על</td>\n",
       "      <td>על</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>780</th>\n",
       "      <td>דחיית</td>\n",
       "      <td>דחייה</td>\n",
       "      <td>דחיית</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>781</th>\n",
       "      <td>כינוס</td>\n",
       "      <td>כינוס</td>\n",
       "      <td>כינוס</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>783</th>\n",
       "      <td>המרכז</td>\n",
       "      <td>מרכז</td>\n",
       "      <td>המרכז</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>785</th>\n",
       "      <td>יו'</td>\n",
       "      <td>יו'</td>\n",
       "      <td>יו'</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>788</th>\n",
       "      <td>העבודה</td>\n",
       "      <td>עבודה</td>\n",
       "      <td>העבודה</td>\n",
       "      <td>I_ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>789</th>\n",
       "      <td>שמעון</td>\n",
       "      <td>שמעון</td>\n",
       "      <td>שמעון</td>\n",
       "      <td>I_PERS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>790</th>\n",
       "      <td>פרס</td>\n",
       "      <td>פרס</td>\n",
       "      <td>פרס</td>\n",
       "      <td>I_PERS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>791</th>\n",
       "      <td>דחה</td>\n",
       "      <td>דחה</td>\n",
       "      <td>דחה</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>793</th>\n",
       "      <td>דברי</td>\n",
       "      <td>דבר</td>\n",
       "      <td>בדברי</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>795</th>\n",
       "      <td>הפתיחה</td>\n",
       "      <td>פתיחה</td>\n",
       "      <td>הפתיחה</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>796</th>\n",
       "      <td>של</td>\n",
       "      <td>של</td>\n",
       "      <td>שלו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>798</th>\n",
       "      <td>את</td>\n",
       "      <td>את</td>\n",
       "      <td>את</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>800</th>\n",
       "      <td>ההצעות</td>\n",
       "      <td>הצעה</td>\n",
       "      <td>ההצעות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>801</th>\n",
       "      <td>כי</td>\n",
       "      <td>כי</td>\n",
       "      <td>כי</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>802</th>\n",
       "      <td>העבודה</td>\n",
       "      <td>עבודה</td>\n",
       "      <td>העבודה</td>\n",
       "      <td>I_ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>803</th>\n",
       "      <td>תנטוש</td>\n",
       "      <td>תנטוש</td>\n",
       "      <td>תנטוש</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>804</th>\n",
       "      <td>את</td>\n",
       "      <td>את</td>\n",
       "      <td>את</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>806</th>\n",
       "      <td>הקואליציה</td>\n",
       "      <td>קואליציה</td>\n",
       "      <td>הקואליציה</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>807</th>\n",
       "      <td>מיד</td>\n",
       "      <td>מייד</td>\n",
       "      <td>מיד</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>808</th>\n",
       "      <td>לאחר</td>\n",
       "      <td>לאחר</td>\n",
       "      <td>לאחר</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>810</th>\n",
       "      <td>ההתנתקות</td>\n",
       "      <td>התנתקות</td>\n",
       "      <td>ההתנתקות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>813</th>\n",
       "      <td>נימק</td>\n",
       "      <td>נמק</td>\n",
       "      <td>ונימק</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Word         Lemma         Token     Bio\n",
       "648          שישה         CARD6          שישה       O\n",
       "650           שטח           שטח          בשטח       O\n",
       "651         ישראל         ישראל         ישראל   I_LOC\n",
       "653         שמונה         CARD8        ושמונה       O\n",
       "655         שטחים           שטח        בשטחים   I_LOC\n",
       "658          דברי           דבר        לדבריו       O\n",
       "661         כוחות           כוח         כוחות       O\n",
       "663       הביטחון        ביטחון       הביטחון       O\n",
       "664         סיכלו          סיכל         סיכלו       O\n",
       "665  ###NUMBER###  ###NUMBER###  ###NUMBER###       O\n",
       "666        פיגועי         פיגוע        פיגועי       O\n",
       "667       התאבדות       התאבדות       התאבדות       O\n",
       "669          תפסו           תפס         ותפסו       O\n",
       "670  ###NUMBER###  ###NUMBER###  ###NUMBER###       O\n",
       "671       מתאבדים         מתאבד       מתאבדים       O\n",
       "673      DOCSTART      DOCSTART      DOCSTART       O\n",
       "676          יהיה            יש          יהיה       O\n",
       "678            נו            נו           לנו       O\n",
       "679          מעמד          מעמד          מעמד       O\n",
       "680          נכבד          נכבד          נכבד       O\n",
       "682         ממשלה         ממשלה        בממשלה       O\n",
       "685           היא           הוא          שהיא       O\n",
       "687         הזירה          זירה         הזירה       O\n",
       "688        אמיתית         אמיתי        אמיתית       O\n",
       "690         ביצוע         ביצוע        לביצוע       O\n",
       "691       שינויים         שינוי       שינויים       O\n",
       "692     משמעותיים       משמעותי     משמעותיים       O\n",
       "694         מישור         מישור        במישור       O\n",
       "696        המדיני         מדיני        המדיני       O\n",
       "699       החברתי'       החברתי'      והחברתי'       O\n",
       "..            ...           ...           ...     ...\n",
       "769         בירור         בירור        לבירור       O\n",
       "770        עתירות         עתירה        עתירות       O\n",
       "771            של            של            של       O\n",
       "773        המפלגה         מפלגה        המפלגה       O\n",
       "774            את            את            את       O\n",
       "775         החלטת         החלטה        החלטתו       O\n",
       "777          מאמש          מאמש          מאמש       O\n",
       "779            על            על            על       O\n",
       "780         דחיית         דחייה         דחיית       O\n",
       "781         כינוס         כינוס         כינוס       O\n",
       "783         המרכז          מרכז         המרכז       O\n",
       "785           יו'           יו'           יו'       O\n",
       "788        העבודה         עבודה        העבודה   I_ORG\n",
       "789         שמעון         שמעון         שמעון  I_PERS\n",
       "790           פרס           פרס           פרס  I_PERS\n",
       "791           דחה           דחה           דחה       O\n",
       "793          דברי           דבר         בדברי       O\n",
       "795        הפתיחה         פתיחה        הפתיחה       O\n",
       "796            של            של           שלו       O\n",
       "798            את            את            את       O\n",
       "800        ההצעות          הצעה        ההצעות       O\n",
       "801            כי            כי            כי       O\n",
       "802        העבודה         עבודה        העבודה   I_ORG\n",
       "803         תנטוש         תנטוש         תנטוש       O\n",
       "804            את            את            את       O\n",
       "806     הקואליציה      קואליציה     הקואליציה       O\n",
       "807           מיד          מייד           מיד       O\n",
       "808          לאחר          לאחר          לאחר       O\n",
       "810      ההתנתקות       התנתקות      ההתנתקות       O\n",
       "813          נימק           נמק         ונימק       O\n",
       "\n",
       "[100 rows x 4 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_df[['Word', 'Lemma', 'Token', 'Bio']][i:i+100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word</th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Token</th>\n",
       "      <th>Bio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>אחד</td>\n",
       "      <td>CARD1</td>\n",
       "      <td>אחד</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>מ</td>\n",
       "      <td>מ</td>\n",
       "      <td>מכל</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>כל</td>\n",
       "      <td>כול</td>\n",
       "      <td>מכל</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>שני</td>\n",
       "      <td>שני</td>\n",
       "      <td>שני</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ישראלים</td>\n",
       "      <td>ישראלי</td>\n",
       "      <td>ישראלים</td>\n",
       "      <td>I_MISC__AFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>אוכלים</td>\n",
       "      <td>אוכל</td>\n",
       "      <td>אוכלים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>שליש</td>\n",
       "      <td>שליש</td>\n",
       "      <td>שליש</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>מן</td>\n",
       "      <td>מן</td>\n",
       "      <td>מהם</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>הם</td>\n",
       "      <td>הם</td>\n",
       "      <td>מהם</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>נהנים</td>\n",
       "      <td>נהנה</td>\n",
       "      <td>נהנים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>מן</td>\n",
       "      <td>מן</td>\n",
       "      <td>ממנו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>נו</td>\n",
       "      <td>נו</td>\n",
       "      <td>ממנו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ב</td>\n",
       "      <td>ב</td>\n",
       "      <td>בבוקר</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>בוקר</td>\n",
       "      <td>בוקר</td>\n",
       "      <td>בבוקר</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>I_PERCENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>אחוז</td>\n",
       "      <td>אחוז</td>\n",
       "      <td>אחוז</td>\n",
       "      <td>I_PERCENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>אוכלים</td>\n",
       "      <td>אוכל</td>\n",
       "      <td>אוכלים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>אותו</td>\n",
       "      <td>אותו</td>\n",
       "      <td>אותו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>ב</td>\n",
       "      <td>ב</td>\n",
       "      <td>בערב</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>ערב</td>\n",
       "      <td>ערב</td>\n",
       "      <td>בערב</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>ו</td>\n",
       "      <td>ו</td>\n",
       "      <td>ואילו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>אילו</td>\n",
       "      <td>אילו</td>\n",
       "      <td>ואילו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>I_PERCENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>%</td>\n",
       "      <td>%</td>\n",
       "      <td>%</td>\n",
       "      <td>I_PERCENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>אוכלים</td>\n",
       "      <td>אוכל</td>\n",
       "      <td>אוכלים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85718</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>I_MONEY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85719</th>\n",
       "      <td>מיליון</td>\n",
       "      <td>ORD-CARD1000000</td>\n",
       "      <td>מיליון</td>\n",
       "      <td>I_MONEY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85720</th>\n",
       "      <td>דולר</td>\n",
       "      <td>דולר</td>\n",
       "      <td>דולר</td>\n",
       "      <td>I_MONEY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85721</th>\n",
       "      <td>ב</td>\n",
       "      <td>ב</td>\n",
       "      <td>בשנת</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85722</th>\n",
       "      <td>שנת</td>\n",
       "      <td>שינה</td>\n",
       "      <td>בשנת</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85723</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>I_DATE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85724</th>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85725</th>\n",
       "      <td>ריטליקס</td>\n",
       "      <td>ריטליקס</td>\n",
       "      <td>ריטליקס</td>\n",
       "      <td>I_ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85726</th>\n",
       "      <td>מפתחת</td>\n",
       "      <td>פיתח</td>\n",
       "      <td>מפתחת</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85727</th>\n",
       "      <td>ו</td>\n",
       "      <td>ו</td>\n",
       "      <td>ומשווקת</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85728</th>\n",
       "      <td>משווקת</td>\n",
       "      <td>שיווק</td>\n",
       "      <td>ומשווקת</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85729</th>\n",
       "      <td>פתרונות</td>\n",
       "      <td>פתרון</td>\n",
       "      <td>פתרונות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85730</th>\n",
       "      <td>תוכנה</td>\n",
       "      <td>תיווך</td>\n",
       "      <td>תוכנה</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85731</th>\n",
       "      <td>כוללים</td>\n",
       "      <td>כולל</td>\n",
       "      <td>כוללים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85732</th>\n",
       "      <td>ל</td>\n",
       "      <td>ל</td>\n",
       "      <td>לתעשיית</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85733</th>\n",
       "      <td>תעשיית</td>\n",
       "      <td>תעשייה</td>\n",
       "      <td>לתעשיית</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85734</th>\n",
       "      <td>קמעונאות</td>\n",
       "      <td>קמעונאות</td>\n",
       "      <td>קמעונאות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85735</th>\n",
       "      <td>ה</td>\n",
       "      <td>ה</td>\n",
       "      <td>המזון</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85736</th>\n",
       "      <td>המזון</td>\n",
       "      <td>מזון</td>\n",
       "      <td>המזון</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85737</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85738</th>\n",
       "      <td>סופרמרקטים</td>\n",
       "      <td>סופרמרקט</td>\n",
       "      <td>סופרמרקטים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85739</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85740</th>\n",
       "      <td>חנויות</td>\n",
       "      <td>חנות</td>\n",
       "      <td>חנויות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85741</th>\n",
       "      <td>נוחות</td>\n",
       "      <td>נוח</td>\n",
       "      <td>נוחות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85742</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85743</th>\n",
       "      <td>תחנות</td>\n",
       "      <td>תחנה</td>\n",
       "      <td>תחנות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85744</th>\n",
       "      <td>דלק</td>\n",
       "      <td>דלק</td>\n",
       "      <td>דלק</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85745</th>\n",
       "      <td>ו</td>\n",
       "      <td>ו</td>\n",
       "      <td>ומסעדות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85746</th>\n",
       "      <td>מסעדות</td>\n",
       "      <td>מסעדה</td>\n",
       "      <td>ומסעדות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85747</th>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>85748 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               Word            Lemma         Token          Bio\n",
       "0          DOCSTART         DOCSTART      DOCSTART            O\n",
       "1               אחד            CARD1           אחד            O\n",
       "2                 מ                מ           מכל            O\n",
       "3                כל              כול           מכל            O\n",
       "4               שני              שני           שני            O\n",
       "5           ישראלים           ישראלי       ישראלים  I_MISC__AFF\n",
       "6            אוכלים             אוכל        אוכלים            O\n",
       "7          קורנפלקס         קורנפלקס      קורנפלקס            O\n",
       "8                 .                .             .            O\n",
       "9              שליש             שליש          שליש            O\n",
       "10               מן               מן           מהם            O\n",
       "11               הם               הם           מהם            O\n",
       "12            נהנים             נהנה         נהנים            O\n",
       "13               מן               מן          ממנו            O\n",
       "14               נו               נו          ממנו            O\n",
       "15                ב                ב         בבוקר            O\n",
       "16             בוקר             בוקר         בבוקר            O\n",
       "17                ,                ,             ,            O\n",
       "18     ###NUMBER###     ###NUMBER###  ###NUMBER###    I_PERCENT\n",
       "19             אחוז             אחוז          אחוז    I_PERCENT\n",
       "20           אוכלים             אוכל        אוכלים            O\n",
       "21             אותו             אותו          אותו            O\n",
       "22                ב                ב          בערב            O\n",
       "23              ערב              ערב          בערב            O\n",
       "24                ,                ,             ,            O\n",
       "25                ו                ו         ואילו            O\n",
       "26             אילו             אילו         ואילו            O\n",
       "27     ###NUMBER###     ###NUMBER###  ###NUMBER###    I_PERCENT\n",
       "28                %                %             %    I_PERCENT\n",
       "29           אוכלים             אוכל        אוכלים            O\n",
       "...             ...              ...           ...          ...\n",
       "85718  ###NUMBER###     ###NUMBER###  ###NUMBER###      I_MONEY\n",
       "85719        מיליון  ORD-CARD1000000        מיליון      I_MONEY\n",
       "85720          דולר             דולר          דולר      I_MONEY\n",
       "85721             ב                ב          בשנת            O\n",
       "85722           שנת             שינה          בשנת            O\n",
       "85723  ###NUMBER###     ###NUMBER###  ###NUMBER###       I_DATE\n",
       "85724             .                .             .            O\n",
       "85725       ריטליקס          ריטליקס       ריטליקס        I_ORG\n",
       "85726         מפתחת             פיתח         מפתחת            O\n",
       "85727             ו                ו       ומשווקת            O\n",
       "85728        משווקת            שיווק       ומשווקת            O\n",
       "85729       פתרונות            פתרון       פתרונות            O\n",
       "85730         תוכנה            תיווך         תוכנה            O\n",
       "85731        כוללים             כולל        כוללים            O\n",
       "85732             ל                ל       לתעשיית            O\n",
       "85733        תעשיית           תעשייה       לתעשיית            O\n",
       "85734      קמעונאות         קמעונאות      קמעונאות            O\n",
       "85735             ה                ה         המזון            O\n",
       "85736         המזון             מזון         המזון            O\n",
       "85737             ,                ,             ,            O\n",
       "85738    סופרמרקטים         סופרמרקט    סופרמרקטים            O\n",
       "85739             ,                ,             ,            O\n",
       "85740        חנויות             חנות        חנויות            O\n",
       "85741         נוחות              נוח         נוחות            O\n",
       "85742             ,                ,             ,            O\n",
       "85743         תחנות             תחנה         תחנות            O\n",
       "85744           דלק              דלק           דלק            O\n",
       "85745             ו                ו       ומסעדות            O\n",
       "85746        מסעדות            מסעדה       ומסעדות            O\n",
       "85747             .                .             .            O\n",
       "\n",
       "[85748 rows x 4 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['Word', 'Lemma', 'Token', 'Bio']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'resources' + os.sep + 'dataset_biluo.csv'    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Token</th>\n",
       "      <th>BILUO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>אחד</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>מכל</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>מכל</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>שני</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Token BILUO\n",
       "0  DOCSTART     O\n",
       "1       אחד     O\n",
       "2       מכל     O\n",
       "3       מכל     O\n",
       "4       שני     O"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['Token', 'BILUO']].head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word</th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Token</th>\n",
       "      <th>BILUO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>אחד</td>\n",
       "      <td>CARD1</td>\n",
       "      <td>אחד</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>מ</td>\n",
       "      <td>מ</td>\n",
       "      <td>מכל</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>כל</td>\n",
       "      <td>כול</td>\n",
       "      <td>מכל</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>שני</td>\n",
       "      <td>שני</td>\n",
       "      <td>שני</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ישראלים</td>\n",
       "      <td>ישראלי</td>\n",
       "      <td>ישראלים</td>\n",
       "      <td>U-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>אוכלים</td>\n",
       "      <td>אוכל</td>\n",
       "      <td>אוכלים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>שליש</td>\n",
       "      <td>שליש</td>\n",
       "      <td>שליש</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>מן</td>\n",
       "      <td>מן</td>\n",
       "      <td>מהם</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>הם</td>\n",
       "      <td>הם</td>\n",
       "      <td>מהם</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>נהנים</td>\n",
       "      <td>נהנה</td>\n",
       "      <td>נהנים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>מן</td>\n",
       "      <td>מן</td>\n",
       "      <td>ממנו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>נו</td>\n",
       "      <td>נו</td>\n",
       "      <td>ממנו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ב</td>\n",
       "      <td>ב</td>\n",
       "      <td>בבוקר</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>בוקר</td>\n",
       "      <td>בוקר</td>\n",
       "      <td>בבוקר</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>B-PERCENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>אחוז</td>\n",
       "      <td>אחוז</td>\n",
       "      <td>אחוז</td>\n",
       "      <td>L-PERCENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>אוכלים</td>\n",
       "      <td>אוכל</td>\n",
       "      <td>אוכלים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>אותו</td>\n",
       "      <td>אותו</td>\n",
       "      <td>אותו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>ב</td>\n",
       "      <td>ב</td>\n",
       "      <td>בערב</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>ערב</td>\n",
       "      <td>ערב</td>\n",
       "      <td>בערב</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>ו</td>\n",
       "      <td>ו</td>\n",
       "      <td>ואילו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>אילו</td>\n",
       "      <td>אילו</td>\n",
       "      <td>ואילו</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>B-PERCENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>%</td>\n",
       "      <td>%</td>\n",
       "      <td>%</td>\n",
       "      <td>L-PERCENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>אוכלים</td>\n",
       "      <td>אוכל</td>\n",
       "      <td>אוכלים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85718</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>B-MONEY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85719</th>\n",
       "      <td>מיליון</td>\n",
       "      <td>ORD-CARD1000000</td>\n",
       "      <td>מיליון</td>\n",
       "      <td>I-MONEY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85720</th>\n",
       "      <td>דולר</td>\n",
       "      <td>דולר</td>\n",
       "      <td>דולר</td>\n",
       "      <td>L-MONEY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85721</th>\n",
       "      <td>ב</td>\n",
       "      <td>ב</td>\n",
       "      <td>בשנת</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85722</th>\n",
       "      <td>שנת</td>\n",
       "      <td>שינה</td>\n",
       "      <td>בשנת</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85723</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>U-DATE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85724</th>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85725</th>\n",
       "      <td>ריטליקס</td>\n",
       "      <td>ריטליקס</td>\n",
       "      <td>ריטליקס</td>\n",
       "      <td>U-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85726</th>\n",
       "      <td>מפתחת</td>\n",
       "      <td>פיתח</td>\n",
       "      <td>מפתחת</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85727</th>\n",
       "      <td>ו</td>\n",
       "      <td>ו</td>\n",
       "      <td>ומשווקת</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85728</th>\n",
       "      <td>משווקת</td>\n",
       "      <td>שיווק</td>\n",
       "      <td>ומשווקת</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85729</th>\n",
       "      <td>פתרונות</td>\n",
       "      <td>פתרון</td>\n",
       "      <td>פתרונות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85730</th>\n",
       "      <td>תוכנה</td>\n",
       "      <td>תיווך</td>\n",
       "      <td>תוכנה</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85731</th>\n",
       "      <td>כוללים</td>\n",
       "      <td>כולל</td>\n",
       "      <td>כוללים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85732</th>\n",
       "      <td>ל</td>\n",
       "      <td>ל</td>\n",
       "      <td>לתעשיית</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85733</th>\n",
       "      <td>תעשיית</td>\n",
       "      <td>תעשייה</td>\n",
       "      <td>לתעשיית</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85734</th>\n",
       "      <td>קמעונאות</td>\n",
       "      <td>קמעונאות</td>\n",
       "      <td>קמעונאות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85735</th>\n",
       "      <td>ה</td>\n",
       "      <td>ה</td>\n",
       "      <td>המזון</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85736</th>\n",
       "      <td>המזון</td>\n",
       "      <td>מזון</td>\n",
       "      <td>המזון</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85737</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85738</th>\n",
       "      <td>סופרמרקטים</td>\n",
       "      <td>סופרמרקט</td>\n",
       "      <td>סופרמרקטים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85739</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85740</th>\n",
       "      <td>חנויות</td>\n",
       "      <td>חנות</td>\n",
       "      <td>חנויות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85741</th>\n",
       "      <td>נוחות</td>\n",
       "      <td>נוח</td>\n",
       "      <td>נוחות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85742</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85743</th>\n",
       "      <td>תחנות</td>\n",
       "      <td>תחנה</td>\n",
       "      <td>תחנות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85744</th>\n",
       "      <td>דלק</td>\n",
       "      <td>דלק</td>\n",
       "      <td>דלק</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85745</th>\n",
       "      <td>ו</td>\n",
       "      <td>ו</td>\n",
       "      <td>ומסעדות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85746</th>\n",
       "      <td>מסעדות</td>\n",
       "      <td>מסעדה</td>\n",
       "      <td>ומסעדות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85747</th>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>85748 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               Word            Lemma         Token      BILUO\n",
       "0          DOCSTART         DOCSTART      DOCSTART          O\n",
       "1               אחד            CARD1           אחד          O\n",
       "2                 מ                מ           מכל          O\n",
       "3                כל              כול           מכל          O\n",
       "4               שני              שני           שני          O\n",
       "5           ישראלים           ישראלי       ישראלים     U-MISC\n",
       "6            אוכלים             אוכל        אוכלים          O\n",
       "7          קורנפלקס         קורנפלקס      קורנפלקס          O\n",
       "8                 .                .             .          O\n",
       "9              שליש             שליש          שליש          O\n",
       "10               מן               מן           מהם          O\n",
       "11               הם               הם           מהם          O\n",
       "12            נהנים             נהנה         נהנים          O\n",
       "13               מן               מן          ממנו          O\n",
       "14               נו               נו          ממנו          O\n",
       "15                ב                ב         בבוקר          O\n",
       "16             בוקר             בוקר         בבוקר          O\n",
       "17                ,                ,             ,          O\n",
       "18     ###NUMBER###     ###NUMBER###  ###NUMBER###  B-PERCENT\n",
       "19             אחוז             אחוז          אחוז  L-PERCENT\n",
       "20           אוכלים             אוכל        אוכלים          O\n",
       "21             אותו             אותו          אותו          O\n",
       "22                ב                ב          בערב          O\n",
       "23              ערב              ערב          בערב          O\n",
       "24                ,                ,             ,          O\n",
       "25                ו                ו         ואילו          O\n",
       "26             אילו             אילו         ואילו          O\n",
       "27     ###NUMBER###     ###NUMBER###  ###NUMBER###  B-PERCENT\n",
       "28                %                %             %  L-PERCENT\n",
       "29           אוכלים             אוכל        אוכלים          O\n",
       "...             ...              ...           ...        ...\n",
       "85718  ###NUMBER###     ###NUMBER###  ###NUMBER###    B-MONEY\n",
       "85719        מיליון  ORD-CARD1000000        מיליון    I-MONEY\n",
       "85720          דולר             דולר          דולר    L-MONEY\n",
       "85721             ב                ב          בשנת          O\n",
       "85722           שנת             שינה          בשנת          O\n",
       "85723  ###NUMBER###     ###NUMBER###  ###NUMBER###     U-DATE\n",
       "85724             .                .             .          O\n",
       "85725       ריטליקס          ריטליקס       ריטליקס      U-ORG\n",
       "85726         מפתחת             פיתח         מפתחת          O\n",
       "85727             ו                ו       ומשווקת          O\n",
       "85728        משווקת            שיווק       ומשווקת          O\n",
       "85729       פתרונות            פתרון       פתרונות          O\n",
       "85730         תוכנה            תיווך         תוכנה          O\n",
       "85731        כוללים             כולל        כוללים          O\n",
       "85732             ל                ל       לתעשיית          O\n",
       "85733        תעשיית           תעשייה       לתעשיית          O\n",
       "85734      קמעונאות         קמעונאות      קמעונאות          O\n",
       "85735             ה                ה         המזון          O\n",
       "85736         המזון             מזון         המזון          O\n",
       "85737             ,                ,             ,          O\n",
       "85738    סופרמרקטים         סופרמרקט    סופרמרקטים          O\n",
       "85739             ,                ,             ,          O\n",
       "85740        חנויות             חנות        חנויות          O\n",
       "85741         נוחות              נוח         נוחות          O\n",
       "85742             ,                ,             ,          O\n",
       "85743         תחנות             תחנה         תחנות          O\n",
       "85744           דלק              דלק           דלק          O\n",
       "85745             ו                ו       ומסעדות          O\n",
       "85746        מסעדות            מסעדה       ומסעדות          O\n",
       "85747             .                .             .        NaN\n",
       "\n",
       "[85748 rows x 4 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['Word', 'Lemma', 'Token', 'BILUO']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df = df[df['Word'].apply(lambda x: len(x) > 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word</th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Token</th>\n",
       "      <th>BILUO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>אבו-ג'וייד</td>\n",
       "      <td>אבו-ג'וייד</td>\n",
       "      <td>אבו-ג'וייד</td>\n",
       "      <td>U-PERS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>התכוון</td>\n",
       "      <td>התכוון</td>\n",
       "      <td>התכוון</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>להקים</td>\n",
       "      <td>הקים</td>\n",
       "      <td>להקים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>חוליות</td>\n",
       "      <td>חוליה</td>\n",
       "      <td>חוליות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>טרור</td>\n",
       "      <td>טרור</td>\n",
       "      <td>טרור</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>בגדה</td>\n",
       "      <td>בגד</td>\n",
       "      <td>בגדה</td>\n",
       "      <td>U-LOC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>בקרב</td>\n",
       "      <td>בקרב</td>\n",
       "      <td>ובקרב</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>ערבי</td>\n",
       "      <td>ערב</td>\n",
       "      <td>ערביי</td>\n",
       "      <td>B-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249</th>\n",
       "      <td>ישראל</td>\n",
       "      <td>ישראל</td>\n",
       "      <td>ישראל</td>\n",
       "      <td>U-LOC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251</th>\n",
       "      <td>לבצע</td>\n",
       "      <td>ביצע</td>\n",
       "      <td>לבצע</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>252</th>\n",
       "      <td>פיגוע</td>\n",
       "      <td>פיגוע</td>\n",
       "      <td>פיגוע</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>רכבת</td>\n",
       "      <td>רכבת</td>\n",
       "      <td>ברכבת</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>ישראל</td>\n",
       "      <td>ישראל</td>\n",
       "      <td>ישראל</td>\n",
       "      <td>L-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>נהריה</td>\n",
       "      <td>נהריה</td>\n",
       "      <td>בנהריה</td>\n",
       "      <td>L-LOC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>259</th>\n",
       "      <td>לפגוע</td>\n",
       "      <td>פגע</td>\n",
       "      <td>לפגוע</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261</th>\n",
       "      <td>מטרות</td>\n",
       "      <td>המטרה</td>\n",
       "      <td>במטרות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>ישראליות</td>\n",
       "      <td>ישראלי</td>\n",
       "      <td>ישראליות</td>\n",
       "      <td>U-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>ירדן</td>\n",
       "      <td>ירדן</td>\n",
       "      <td>בירדן</td>\n",
       "      <td>L-LOC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>לחטוף</td>\n",
       "      <td>חטף</td>\n",
       "      <td>ולחטוף</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>חיילים</td>\n",
       "      <td>חייל</td>\n",
       "      <td>חיילים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269</th>\n",
       "      <td>די</td>\n",
       "      <td>די</td>\n",
       "      <td>כדי</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270</th>\n",
       "      <td>לשחרר</td>\n",
       "      <td>שחרר</td>\n",
       "      <td>לשחרר</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>271</th>\n",
       "      <td>אסירים</td>\n",
       "      <td>אסיר</td>\n",
       "      <td>אסירים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272</th>\n",
       "      <td>ביטחוניים</td>\n",
       "      <td>בטחוני</td>\n",
       "      <td>ביטחוניים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274</th>\n",
       "      <td>אבו-ג'וייד</td>\n",
       "      <td>אבו-ג'וייד</td>\n",
       "      <td>אבו-ג'וייד</td>\n",
       "      <td>U-PERS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>276</th>\n",
       "      <td>נולד</td>\n",
       "      <td>נולד</td>\n",
       "      <td>נולד</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278</th>\n",
       "      <td>סוריה</td>\n",
       "      <td>סוריה</td>\n",
       "      <td>בסוריה</td>\n",
       "      <td>L-LOC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>גיל</td>\n",
       "      <td>גיל</td>\n",
       "      <td>ובגיל</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>284</th>\n",
       "      <td>עבר</td>\n",
       "      <td>עבר</td>\n",
       "      <td>עבר</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1561</th>\n",
       "      <td>את</td>\n",
       "      <td>את</td>\n",
       "      <td>את</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1563</th>\n",
       "      <td>העיר</td>\n",
       "      <td>עיר</td>\n",
       "      <td>העיר</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1565</th>\n",
       "      <td>הגדולה</td>\n",
       "      <td>גדולה</td>\n",
       "      <td>הגדולה</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1566</th>\n",
       "      <td>עד</td>\n",
       "      <td>עד</td>\n",
       "      <td>עד</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1568</th>\n",
       "      <td>מרגלות</td>\n",
       "      <td>מרגלות</td>\n",
       "      <td>למרגלות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1570</th>\n",
       "      <td>הפירמידות</td>\n",
       "      <td>פירמידה</td>\n",
       "      <td>הפירמידות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1573</th>\n",
       "      <td>ערב</td>\n",
       "      <td>ערב</td>\n",
       "      <td>בערב</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1574</th>\n",
       "      <td>מתקיים</td>\n",
       "      <td>התקיים</td>\n",
       "      <td>מתקיים</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1575</th>\n",
       "      <td>מופע</td>\n",
       "      <td>מופע</td>\n",
       "      <td>מופע</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1576</th>\n",
       "      <td>אור-קולי</td>\n",
       "      <td>אור-קולי</td>\n",
       "      <td>אור-קולי</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1578</th>\n",
       "      <td>אזור</td>\n",
       "      <td>אזור</td>\n",
       "      <td>באזור</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1580</th>\n",
       "      <td>הפירמידות</td>\n",
       "      <td>פירמידה</td>\n",
       "      <td>הפירמידות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1582</th>\n",
       "      <td>גם</td>\n",
       "      <td>גם</td>\n",
       "      <td>גם</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1584</th>\n",
       "      <td>מי</td>\n",
       "      <td>מי</td>\n",
       "      <td>למי</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1586</th>\n",
       "      <td>מוותר</td>\n",
       "      <td>ויתר</td>\n",
       "      <td>שמוותר</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1587</th>\n",
       "      <td>על</td>\n",
       "      <td>על</td>\n",
       "      <td>על</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1589</th>\n",
       "      <td>המופע</td>\n",
       "      <td>מופע</td>\n",
       "      <td>המופע</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1590</th>\n",
       "      <td>מומלץ</td>\n",
       "      <td>מומלץ</td>\n",
       "      <td>מומלץ</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1591</th>\n",
       "      <td>לעבור</td>\n",
       "      <td>עבר</td>\n",
       "      <td>לעבור</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1593</th>\n",
       "      <td>אזור</td>\n",
       "      <td>אזור</td>\n",
       "      <td>באזור</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1596</th>\n",
       "      <td>ראות</td>\n",
       "      <td>ראות</td>\n",
       "      <td>ולראות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1597</th>\n",
       "      <td>את</td>\n",
       "      <td>את</td>\n",
       "      <td>את</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1599</th>\n",
       "      <td>התאורה</td>\n",
       "      <td>תאורה</td>\n",
       "      <td>התאורה</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1601</th>\n",
       "      <td>המרשימה</td>\n",
       "      <td>מרשים</td>\n",
       "      <td>המרשימה</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1603</th>\n",
       "      <td>מאירה</td>\n",
       "      <td>האיר</td>\n",
       "      <td>שמאירה</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1604</th>\n",
       "      <td>את</td>\n",
       "      <td>את</td>\n",
       "      <td>את</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1606</th>\n",
       "      <td>הפירמידות</td>\n",
       "      <td>פירמידה</td>\n",
       "      <td>הפירמידות</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1608</th>\n",
       "      <td>טיול</td>\n",
       "      <td>טיול</td>\n",
       "      <td>טיול</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1610</th>\n",
       "      <td>עניין</td>\n",
       "      <td>עניין</td>\n",
       "      <td>מעניין</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1611</th>\n",
       "      <td>נוסף</td>\n",
       "      <td>נוסף</td>\n",
       "      <td>נוסף</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>850 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              Word         Lemma         Token   BILUO\n",
       "239     אבו-ג'וייד    אבו-ג'וייד    אבו-ג'וייד  U-PERS\n",
       "240         התכוון        התכוון        התכוון       O\n",
       "241          להקים          הקים         להקים       O\n",
       "242         חוליות         חוליה        חוליות       O\n",
       "243           טרור          טרור          טרור       O\n",
       "244           בגדה           בגד          בגדה   U-LOC\n",
       "246           בקרב          בקרב         ובקרב       O\n",
       "247           ערבי           ערב         ערביי  B-MISC\n",
       "249          ישראל         ישראל         ישראל   U-LOC\n",
       "251           לבצע          ביצע          לבצע       O\n",
       "252          פיגוע         פיגוע         פיגוע       O\n",
       "254           רכבת          רכבת         ברכבת   I-ORG\n",
       "255          ישראל         ישראל         ישראל   L-ORG\n",
       "257          נהריה         נהריה        בנהריה   L-LOC\n",
       "259          לפגוע           פגע         לפגוע       O\n",
       "261          מטרות         המטרה        במטרות       O\n",
       "262       ישראליות        ישראלי      ישראליות  U-MISC\n",
       "264           ירדן          ירדן         בירדן   L-LOC\n",
       "266          לחטוף           חטף        ולחטוף       O\n",
       "267         חיילים          חייל        חיילים       O\n",
       "269             די            די           כדי       O\n",
       "270          לשחרר          שחרר         לשחרר       O\n",
       "271         אסירים          אסיר        אסירים       O\n",
       "272      ביטחוניים        בטחוני     ביטחוניים       O\n",
       "274     אבו-ג'וייד    אבו-ג'וייד    אבו-ג'וייד  U-PERS\n",
       "276           נולד          נולד          נולד       O\n",
       "278          סוריה         סוריה        בסוריה   L-LOC\n",
       "282            גיל           גיל         ובגיל       O\n",
       "283   ###NUMBER###  ###NUMBER###  ###NUMBER###       O\n",
       "284            עבר           עבר           עבר       O\n",
       "...            ...           ...           ...     ...\n",
       "1561            את            את            את       O\n",
       "1563          העיר           עיר          העיר       O\n",
       "1565        הגדולה         גדולה        הגדולה       O\n",
       "1566            עד            עד            עד       O\n",
       "1568        מרגלות        מרגלות       למרגלות       O\n",
       "1570     הפירמידות       פירמידה     הפירמידות       O\n",
       "1573           ערב           ערב          בערב       O\n",
       "1574        מתקיים        התקיים        מתקיים       O\n",
       "1575          מופע          מופע          מופע       O\n",
       "1576      אור-קולי      אור-קולי      אור-קולי       O\n",
       "1578          אזור          אזור         באזור       O\n",
       "1580     הפירמידות       פירמידה     הפירמידות       O\n",
       "1582            גם            גם            גם       O\n",
       "1584            מי            מי           למי       O\n",
       "1586         מוותר          ויתר        שמוותר       O\n",
       "1587            על            על            על       O\n",
       "1589         המופע          מופע         המופע       O\n",
       "1590         מומלץ         מומלץ         מומלץ       O\n",
       "1591         לעבור           עבר         לעבור       O\n",
       "1593          אזור          אזור         באזור       O\n",
       "1596          ראות          ראות        ולראות       O\n",
       "1597            את            את            את       O\n",
       "1599        התאורה         תאורה        התאורה       O\n",
       "1601       המרשימה         מרשים       המרשימה       O\n",
       "1603         מאירה          האיר        שמאירה       O\n",
       "1604            את            את            את       O\n",
       "1606     הפירמידות       פירמידה     הפירמידות       O\n",
       "1608          טיול          טיול          טיול       O\n",
       "1610         עניין         עניין        מעניין       O\n",
       "1611          נוסף          נוסף          נוסף       O\n",
       "\n",
       "[850 rows x 4 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_df[['Word', 'Lemma', 'Token', 'BILUO']][150:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-MISC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-PERCENT',\n",
       " 'L-PERCENT',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-PERCENT',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-MISC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-PERCENT',\n",
       " 'L-MISC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-PERCENT',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-ORG',\n",
       " 'L-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-PERCENT',\n",
       " 'L-MISC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-PERCENT',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-PERCENT',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-MISC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-DATE',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'B-PERS',\n",
       " 'L-PERS',\n",
       " 'O',\n",
       " 'U-MISC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-LOC',\n",
       " 'O',\n",
       " 'B-MISC',\n",
       " 'U-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'I-ORG',\n",
       " 'L-ORG',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-MISC',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'U-DATE',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'L-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-MISC',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-MISC',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-MISC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-MISC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-ORG',\n",
       " 'O',\n",
       " 'L-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-MISC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-MISC',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'B-MISC',\n",
       " 'U-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-MISC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-LOC',\n",
       " 'I-DATE',\n",
       " 'L-DATE',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-DATE',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-PERCENT',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-PERCENT',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'I-ORG',\n",
       " 'I-ORG',\n",
       " 'L-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-LOC',\n",
       " 'O',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'B-PERS',\n",
       " 'L-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'I-LOC',\n",
       " 'I-LOC',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'B-PERS',\n",
       " 'L-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-PERS',\n",
       " 'L-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-PERS',\n",
       " 'L-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-ORG',\n",
       " 'L-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'I-ORG',\n",
       " 'L-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'I-ORG',\n",
       " 'L-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-ORG',\n",
       " 'I-ORG',\n",
       " 'I-ORG',\n",
       " 'L-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'I-ORG',\n",
       " 'I-ORG',\n",
       " 'I-ORG',\n",
       " 'L-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-ORG',\n",
       " 'O',\n",
       " 'I-ORG',\n",
       " 'I-ORG',\n",
       " 'I-ORG',\n",
       " 'I-ORG',\n",
       " 'L-ORG',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'I-MISC',\n",
       " 'L-MISC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'L-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-PERS',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'B-DATE',\n",
       " 'I-DATE',\n",
       " 'L-DATE',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'U-LOC',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " 'O',\n",
       " ...]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l = list(filtered_df['BILUO'].values)\n",
    "l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "i_misc_data = df[df['BILUO'] == 'I-MISC']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's observe the data, see if we can find interesting features that seperates them from the rest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bio</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Number</th>\n",
       "      <th>Person</th>\n",
       "      <th>Pos</th>\n",
       "      <th>Status</th>\n",
       "      <th>Tense</th>\n",
       "      <th>Token</th>\n",
       "      <th>TokenOrder</th>\n",
       "      <th>Word</th>\n",
       "      <th>BILUO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>781</th>\n",
       "      <td>I_MISC__AFF</td>\n",
       "      <td>masculine</td>\n",
       "      <td>יווני</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>adjective</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>היוונים</td>\n",
       "      <td>1149</td>\n",
       "      <td>היוונים</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3426</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ן</td>\n",
       "      <td>plural</td>\n",
       "      <td>3</td>\n",
       "      <td>possessive</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>צופן</td>\n",
       "      <td>4658</td>\n",
       "      <td>ן</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3427</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>דה</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>דה</td>\n",
       "      <td>4659</td>\n",
       "      <td>דה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3503</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ן</td>\n",
       "      <td>plural</td>\n",
       "      <td>3</td>\n",
       "      <td>possessive</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>צופן</td>\n",
       "      <td>4753</td>\n",
       "      <td>ן</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3504</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>דה</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>דה</td>\n",
       "      <td>4754</td>\n",
       "      <td>דה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3666</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>סוד</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הסודות</td>\n",
       "      <td>4971</td>\n",
       "      <td>הסודות</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3667</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>צפה</td>\n",
       "      <td>singular</td>\n",
       "      <td>any</td>\n",
       "      <td>participle</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>צופן</td>\n",
       "      <td>4973</td>\n",
       "      <td>צוף</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3668</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ן</td>\n",
       "      <td>plural</td>\n",
       "      <td>3</td>\n",
       "      <td>possessive</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>צופן</td>\n",
       "      <td>4973</td>\n",
       "      <td>ן</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3669</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>דה</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>דה</td>\n",
       "      <td>4974</td>\n",
       "      <td>דה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3708</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ברית</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בברית</td>\n",
       "      <td>5025</td>\n",
       "      <td>ברית</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3778</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>גילה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>מגילת</td>\n",
       "      <td>5123</td>\n",
       "      <td>גילת</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3800</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ברית</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בברית</td>\n",
       "      <td>5155</td>\n",
       "      <td>ברית</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3937</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>סוד</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הסודות</td>\n",
       "      <td>5361</td>\n",
       "      <td>הסודות</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3938</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>צפה</td>\n",
       "      <td>singular</td>\n",
       "      <td>any</td>\n",
       "      <td>participle</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>צופן</td>\n",
       "      <td>5363</td>\n",
       "      <td>צוף</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3939</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ן</td>\n",
       "      <td>plural</td>\n",
       "      <td>3</td>\n",
       "      <td>possessive</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>צופן</td>\n",
       "      <td>5363</td>\n",
       "      <td>ן</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3940</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>דה</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>דה</td>\n",
       "      <td>5364</td>\n",
       "      <td>דה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4419</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>דם</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הדם</td>\n",
       "      <td>6039</td>\n",
       "      <td>הדם</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4420</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>קדוש</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הקדוש</td>\n",
       "      <td>6040</td>\n",
       "      <td>הקדוש</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4421</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>גביע</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>והגביע</td>\n",
       "      <td>6041</td>\n",
       "      <td>הגביע</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4598</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ן</td>\n",
       "      <td>plural</td>\n",
       "      <td>3</td>\n",
       "      <td>possessive</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>צופן</td>\n",
       "      <td>6293</td>\n",
       "      <td>ן</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Bio       Gender  Lemma       Number       Person         Pos  \\\n",
       "781   I_MISC__AFF    masculine  יווני       plural  unspecified   adjective   \n",
       "3426  I_MISC__ENT     feminine      ן       plural            3  possessive   \n",
       "3427  I_MISC__ENT  unspecified     דה  unspecified  unspecified  properName   \n",
       "3503  I_MISC__ENT     feminine      ן       plural            3  possessive   \n",
       "3504  I_MISC__ENT  unspecified     דה  unspecified  unspecified  properName   \n",
       "3666  I_MISC__ENT    masculine    סוד       plural  unspecified        noun   \n",
       "3667  I_MISC__ENT    masculine    צפה     singular          any  participle   \n",
       "3668  I_MISC__ENT     feminine      ן       plural            3  possessive   \n",
       "3669  I_MISC__ENT  unspecified     דה  unspecified  unspecified  properName   \n",
       "3708  I_MISC__ENT     feminine   ברית     singular  unspecified        noun   \n",
       "3778  I_MISC__ENT     feminine   גילה     singular  unspecified        noun   \n",
       "3800  I_MISC__ENT     feminine   ברית     singular  unspecified        noun   \n",
       "3937  I_MISC__ENT    masculine    סוד       plural  unspecified        noun   \n",
       "3938  I_MISC__ENT    masculine    צפה     singular          any  participle   \n",
       "3939  I_MISC__ENT     feminine      ן       plural            3  possessive   \n",
       "3940  I_MISC__ENT  unspecified     דה  unspecified  unspecified  properName   \n",
       "4419  I_MISC__ENT    masculine     דם     singular  unspecified        noun   \n",
       "4420  I_MISC__ENT    masculine   קדוש     singular  unspecified        noun   \n",
       "4421  I_MISC__ENT    masculine   גביע     singular  unspecified        noun   \n",
       "4598  I_MISC__ENT     feminine      ן       plural            3  possessive   \n",
       "\n",
       "           Status                Tense    Token  TokenOrder     Word   BILUO  \n",
       "781      absolute  unspecified\\r\\r\\r\\n  היוונים        1149  היוונים  I-MISC  \n",
       "3426  unspecified  unspecified\\r\\r\\r\\n     צופן        4658        ן  I-MISC  \n",
       "3427  unspecified  unspecified\\r\\r\\r\\n       דה        4659       דה  I-MISC  \n",
       "3503  unspecified  unspecified\\r\\r\\r\\n     צופן        4753        ן  I-MISC  \n",
       "3504  unspecified  unspecified\\r\\r\\r\\n       דה        4754       דה  I-MISC  \n",
       "3666     absolute  unspecified\\r\\r\\r\\n   הסודות        4971   הסודות  I-MISC  \n",
       "3667  unspecified  unspecified\\r\\r\\r\\n     צופן        4973      צוף  I-MISC  \n",
       "3668  unspecified  unspecified\\r\\r\\r\\n     צופן        4973        ן  I-MISC  \n",
       "3669  unspecified  unspecified\\r\\r\\r\\n       דה        4974       דה  I-MISC  \n",
       "3708     absolute  unspecified\\r\\r\\r\\n    בברית        5025     ברית  I-MISC  \n",
       "3778    construct  unspecified\\r\\r\\r\\n    מגילת        5123     גילת  I-MISC  \n",
       "3800     absolute  unspecified\\r\\r\\r\\n    בברית        5155     ברית  I-MISC  \n",
       "3937     absolute  unspecified\\r\\r\\r\\n   הסודות        5361   הסודות  I-MISC  \n",
       "3938  unspecified  unspecified\\r\\r\\r\\n     צופן        5363      צוף  I-MISC  \n",
       "3939  unspecified  unspecified\\r\\r\\r\\n     צופן        5363        ן  I-MISC  \n",
       "3940  unspecified  unspecified\\r\\r\\r\\n       דה        5364       דה  I-MISC  \n",
       "4419     absolute  unspecified\\r\\r\\r\\n      הדם        6039      הדם  I-MISC  \n",
       "4420     absolute  unspecified\\r\\r\\r\\n    הקדוש        6040    הקדוש  I-MISC  \n",
       "4421     absolute  unspecified\\r\\r\\r\\n   והגביע        6041    הגביע  I-MISC  \n",
       "4598  unspecified  unspecified\\r\\r\\r\\n     צופן        6293        ן  I-MISC  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i_misc_data.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not finding something special :-("
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "bad_perf = ['I-ORG', 'I-MISC', 'I-TIME', 'L-TIME']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bio</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Number</th>\n",
       "      <th>Person</th>\n",
       "      <th>Pos</th>\n",
       "      <th>Status</th>\n",
       "      <th>Tense</th>\n",
       "      <th>Token</th>\n",
       "      <th>TokenOrder</th>\n",
       "      <th>Word</th>\n",
       "      <th>BILUO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>רכבת</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ברכבת</td>\n",
       "      <td>298</td>\n",
       "      <td>רכבת</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>372</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ועדה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בוועדת</td>\n",
       "      <td>582</td>\n",
       "      <td>וועדת</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>373</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>חוץ</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>החוץ</td>\n",
       "      <td>583</td>\n",
       "      <td>החוץ</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>659</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>משרד</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>במשרד</td>\n",
       "      <td>969</td>\n",
       "      <td>משרד</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>671</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>משרד</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>במשרד</td>\n",
       "      <td>985</td>\n",
       "      <td>משרד</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>693</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>מם</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>מ\"מ</td>\n",
       "      <td>1010</td>\n",
       "      <td>מ\"מ</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>699</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ועדה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בוועדה</td>\n",
       "      <td>1019</td>\n",
       "      <td>וועדה</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>700</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>מינוי</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>למינוי</td>\n",
       "      <td>1020</td>\n",
       "      <td>מינוי</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>701</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>שופט</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>שופטים</td>\n",
       "      <td>1021</td>\n",
       "      <td>שופטים</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>721</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ועדה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בוועדת</td>\n",
       "      <td>1044</td>\n",
       "      <td>וועדת</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>722</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>שר</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>השרים</td>\n",
       "      <td>1045</td>\n",
       "      <td>השרים</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>723</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>הרכב</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>להרכבי</td>\n",
       "      <td>1046</td>\n",
       "      <td>הרכבי</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>724</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>מועצה</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>המועצות</td>\n",
       "      <td>1047</td>\n",
       "      <td>המועצות</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1964</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>רואה</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>רואי</td>\n",
       "      <td>2702</td>\n",
       "      <td>רואי</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2261</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>פועל</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>להפועל</td>\n",
       "      <td>3132</td>\n",
       "      <td>הפועל</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2263</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>פועל</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>להפועל</td>\n",
       "      <td>3135</td>\n",
       "      <td>הפועל</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2933</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>איחוד</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>האיחוד</td>\n",
       "      <td>4015</td>\n",
       "      <td>האיחוד</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3083</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ישיבה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>לישיבת</td>\n",
       "      <td>4221</td>\n",
       "      <td>ישיבת</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3465</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>ִילו</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ִילו</td>\n",
       "      <td>4707</td>\n",
       "      <td>ִילו</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3466</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>ּמי</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ּמינ</td>\n",
       "      <td>4708</td>\n",
       "      <td>ּמי</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3485</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>כנסייה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הכנסייה</td>\n",
       "      <td>4730</td>\n",
       "      <td>הכנסייה</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3557</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>סדר</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>מסדר</td>\n",
       "      <td>4822</td>\n",
       "      <td>סדר</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3571</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>האו</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>האו</td>\n",
       "      <td>4839</td>\n",
       "      <td>האו</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3572</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>ֹ</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ֹפ</td>\n",
       "      <td>4840</td>\n",
       "      <td>ֹ</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3573</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>ּו</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ּו</td>\n",
       "      <td>4841</td>\n",
       "      <td>ּו</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3574</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>ּס</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ּס</td>\n",
       "      <td>4842</td>\n",
       "      <td>ּס</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3575</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>ֵא</td>\n",
       "      <td>singular</td>\n",
       "      <td>3</td>\n",
       "      <td>verb</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>past\\r\\r\\r\\n</td>\n",
       "      <td>ֵא</td>\n",
       "      <td>4844</td>\n",
       "      <td>ֵא</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4008</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>סדר</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>מסדר</td>\n",
       "      <td>5468</td>\n",
       "      <td>סדר</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4084</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>ֶמף</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ֶמפל</td>\n",
       "      <td>5572</td>\n",
       "      <td>ֶמף</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4166</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>בונה</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הבונים</td>\n",
       "      <td>5683</td>\n",
       "      <td>הבונים</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44284</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>לשכה</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>לשכות</td>\n",
       "      <td>60537</td>\n",
       "      <td>לשכות</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44678</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>משרד</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>למשרד</td>\n",
       "      <td>61048</td>\n",
       "      <td>משרד</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44679</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>אוצר</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>האוצר</td>\n",
       "      <td>61049</td>\n",
       "      <td>האוצר</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44680</th>\n",
       "      <td>B_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>משרד</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ולמשרד</td>\n",
       "      <td>61050</td>\n",
       "      <td>משרד</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44703</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>התאחדות</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בהתאחדות</td>\n",
       "      <td>61082</td>\n",
       "      <td>התאחדות</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44849</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>-</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>-</td>\n",
       "      <td>61259</td>\n",
       "      <td>-</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44850</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>סב</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>סבן</td>\n",
       "      <td>61260</td>\n",
       "      <td>סבן</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44851</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>-</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>-</td>\n",
       "      <td>61261</td>\n",
       "      <td>-</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45214</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>בנק</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בבנק</td>\n",
       "      <td>61726</td>\n",
       "      <td>בנק</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45704</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>חיתום</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>חיתום</td>\n",
       "      <td>62322</td>\n",
       "      <td>חיתום</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45705</th>\n",
       "      <td>B_ORG</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>ורוסאריו</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ורוסאריו</td>\n",
       "      <td>62323</td>\n",
       "      <td>ורוסאריו</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45854</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>קבוצה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בקבוצת</td>\n",
       "      <td>62498</td>\n",
       "      <td>קבוצת</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45859</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>חיפוש</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>חיפושי</td>\n",
       "      <td>62504</td>\n",
       "      <td>חיפושי</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45970</th>\n",
       "      <td>B_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>קבוצה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>מקבוצת</td>\n",
       "      <td>62637</td>\n",
       "      <td>קבוצת</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46057</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>-</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>-</td>\n",
       "      <td>62743</td>\n",
       "      <td>-</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46067</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>-</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>punctuation</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>-</td>\n",
       "      <td>62755</td>\n",
       "      <td>-</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46074</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>תעשייה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>התעשיה</td>\n",
       "      <td>62763</td>\n",
       "      <td>התעשיה</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46195</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>Venture</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>Venture</td>\n",
       "      <td>62909</td>\n",
       "      <td>Venture</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46196</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>Venture</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>Venture</td>\n",
       "      <td>62913</td>\n",
       "      <td>Venture</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46197</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>Capital</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>Capital</td>\n",
       "      <td>62914</td>\n",
       "      <td>Capital</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46349</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>איחוד</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>לאיחוד</td>\n",
       "      <td>63105</td>\n",
       "      <td>איחוד</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46373</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>מיכון</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>במכון</td>\n",
       "      <td>63133</td>\n",
       "      <td>מכון</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46410</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>איחוד</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>האיחוד</td>\n",
       "      <td>63178</td>\n",
       "      <td>האיחוד</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46452</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>איחוד</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>מהאיחוד</td>\n",
       "      <td>63226</td>\n",
       "      <td>האיחוד</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46482</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>איחוד</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>האיחוד</td>\n",
       "      <td>63263</td>\n",
       "      <td>האיחוד</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46489</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>איחוד</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>באיחוד</td>\n",
       "      <td>63274</td>\n",
       "      <td>איחוד</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46528</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>masculine</td>\n",
       "      <td>איחוד</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>לאיחוד</td>\n",
       "      <td>63324</td>\n",
       "      <td>איחוד</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46567</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>לשכה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הלשכה</td>\n",
       "      <td>63374</td>\n",
       "      <td>הלשכה</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46568</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>feminine</td>\n",
       "      <td>לאומי</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>adjective</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הלאומית</td>\n",
       "      <td>63375</td>\n",
       "      <td>הלאומית</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46574</th>\n",
       "      <td>I_ORG</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>להמן</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בלהמן</td>\n",
       "      <td>63381</td>\n",
       "      <td>להמן</td>\n",
       "      <td>I-ORG</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>477 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Bio       Gender     Lemma       Number       Person          Pos  \\\n",
       "150    I_ORG     feminine      רכבת     singular  unspecified         noun   \n",
       "372    I_ORG     feminine      ועדה     singular  unspecified         noun   \n",
       "373    I_ORG    masculine       חוץ     singular  unspecified         noun   \n",
       "659    I_ORG    masculine      משרד     singular  unspecified         noun   \n",
       "671    I_ORG    masculine      משרד     singular  unspecified         noun   \n",
       "693    I_ORG     feminine        מם     singular  unspecified   properName   \n",
       "699    I_ORG     feminine      ועדה     singular  unspecified         noun   \n",
       "700    I_ORG    masculine     מינוי     singular  unspecified         noun   \n",
       "701    I_ORG    masculine      שופט       plural  unspecified         noun   \n",
       "721    I_ORG     feminine      ועדה     singular  unspecified         noun   \n",
       "722    I_ORG    masculine        שר       plural  unspecified         noun   \n",
       "723    I_ORG    masculine      הרכב       plural  unspecified         noun   \n",
       "724    I_ORG     feminine     מועצה       plural  unspecified         noun   \n",
       "1964   I_ORG    masculine      רואה       plural  unspecified         noun   \n",
       "2261   I_ORG  unspecified      פועל  unspecified  unspecified   properName   \n",
       "2263   I_ORG  unspecified      פועל  unspecified  unspecified   properName   \n",
       "2933   I_ORG    masculine     איחוד     singular  unspecified         noun   \n",
       "3083   I_ORG     feminine     ישיבה     singular  unspecified         noun   \n",
       "3465   I_ORG  unspecified      ִילו  unspecified  unspecified   properName   \n",
       "3466   I_ORG    masculine       ּמי     singular  unspecified         noun   \n",
       "3485   I_ORG     feminine    כנסייה     singular  unspecified         noun   \n",
       "3557   I_ORG    masculine       סדר     singular  unspecified         noun   \n",
       "3571   I_ORG     feminine       האו     singular  unspecified         noun   \n",
       "3572   I_ORG    masculine         ֹ     singular  unspecified         noun   \n",
       "3573   I_ORG    masculine        ּו     singular  unspecified         noun   \n",
       "3574   I_ORG  unspecified        ּס  unspecified  unspecified   properName   \n",
       "3575   I_ORG    masculine        ֵא     singular            3         verb   \n",
       "4008   I_ORG    masculine       סדר     singular  unspecified         noun   \n",
       "4084   I_ORG    masculine       ֶמף     singular  unspecified         noun   \n",
       "4166   I_ORG    masculine      בונה       plural  unspecified         noun   \n",
       "...      ...          ...       ...          ...          ...          ...   \n",
       "44284  I_ORG     feminine      לשכה       plural  unspecified         noun   \n",
       "44678  I_ORG    masculine      משרד     singular  unspecified         noun   \n",
       "44679  I_ORG    masculine      אוצר     singular  unspecified         noun   \n",
       "44680  B_ORG    masculine      משרד     singular  unspecified         noun   \n",
       "44703  I_ORG     feminine   התאחדות     singular  unspecified         noun   \n",
       "44849  I_ORG  unspecified         -  unspecified  unspecified   properName   \n",
       "44850  I_ORG  unspecified        סב  unspecified  unspecified   properName   \n",
       "44851  I_ORG  unspecified         -  unspecified  unspecified   properName   \n",
       "45214  I_ORG    masculine       בנק     singular  unspecified         noun   \n",
       "45704  I_ORG    masculine     חיתום     singular  unspecified         noun   \n",
       "45705  B_ORG  unspecified  ורוסאריו  unspecified  unspecified   properName   \n",
       "45854  I_ORG     feminine     קבוצה     singular  unspecified         noun   \n",
       "45859  I_ORG    masculine     חיפוש       plural  unspecified         noun   \n",
       "45970  B_ORG     feminine     קבוצה     singular  unspecified         noun   \n",
       "46057  I_ORG  unspecified         -  unspecified  unspecified   properName   \n",
       "46067  I_ORG  unspecified         -  unspecified  unspecified  punctuation   \n",
       "46074  I_ORG     feminine    תעשייה     singular  unspecified         noun   \n",
       "46195  I_ORG  unspecified   Venture  unspecified  unspecified   properName   \n",
       "46196  I_ORG  unspecified   Venture  unspecified  unspecified   properName   \n",
       "46197  I_ORG  unspecified   Capital  unspecified  unspecified   properName   \n",
       "46349  I_ORG    masculine     איחוד     singular  unspecified         noun   \n",
       "46373  I_ORG    masculine     מיכון     singular  unspecified         noun   \n",
       "46410  I_ORG    masculine     איחוד     singular  unspecified         noun   \n",
       "46452  I_ORG    masculine     איחוד     singular  unspecified         noun   \n",
       "46482  I_ORG    masculine     איחוד     singular  unspecified         noun   \n",
       "46489  I_ORG    masculine     איחוד     singular  unspecified         noun   \n",
       "46528  I_ORG    masculine     איחוד     singular  unspecified         noun   \n",
       "46567  I_ORG     feminine      לשכה     singular  unspecified         noun   \n",
       "46568  I_ORG     feminine     לאומי     singular  unspecified    adjective   \n",
       "46574  I_ORG  unspecified      להמן  unspecified  unspecified   properName   \n",
       "\n",
       "            Status                Tense     Token  TokenOrder      Word  BILUO  \n",
       "150       absolute  unspecified\\r\\r\\r\\n     ברכבת         298      רכבת  I-ORG  \n",
       "372      construct  unspecified\\r\\r\\r\\n    בוועדת         582     וועדת  I-ORG  \n",
       "373       absolute  unspecified\\r\\r\\r\\n      החוץ         583      החוץ  I-ORG  \n",
       "659       absolute  unspecified\\r\\r\\r\\n     במשרד         969      משרד  I-ORG  \n",
       "671       absolute  unspecified\\r\\r\\r\\n     במשרד         985      משרד  I-ORG  \n",
       "693    unspecified  unspecified\\r\\r\\r\\n       מ\"מ        1010       מ\"מ  I-ORG  \n",
       "699       absolute  unspecified\\r\\r\\r\\n    בוועדה        1019     וועדה  I-ORG  \n",
       "700       absolute  unspecified\\r\\r\\r\\n    למינוי        1020     מינוי  I-ORG  \n",
       "701       absolute  unspecified\\r\\r\\r\\n    שופטים        1021    שופטים  I-ORG  \n",
       "721      construct  unspecified\\r\\r\\r\\n    בוועדת        1044     וועדת  I-ORG  \n",
       "722       absolute  unspecified\\r\\r\\r\\n     השרים        1045     השרים  I-ORG  \n",
       "723      construct  unspecified\\r\\r\\r\\n    להרכבי        1046     הרכבי  I-ORG  \n",
       "724       absolute  unspecified\\r\\r\\r\\n   המועצות        1047   המועצות  I-ORG  \n",
       "1964     construct  unspecified\\r\\r\\r\\n      רואי        2702      רואי  I-ORG  \n",
       "2261   unspecified  unspecified\\r\\r\\r\\n    להפועל        3132     הפועל  I-ORG  \n",
       "2263   unspecified  unspecified\\r\\r\\r\\n    להפועל        3135     הפועל  I-ORG  \n",
       "2933      absolute  unspecified\\r\\r\\r\\n    האיחוד        4015    האיחוד  I-ORG  \n",
       "3083     construct  unspecified\\r\\r\\r\\n    לישיבת        4221     ישיבת  I-ORG  \n",
       "3465   unspecified  unspecified\\r\\r\\r\\n      ִילו        4707      ִילו  I-ORG  \n",
       "3466      absolute  unspecified\\r\\r\\r\\n      ּמינ        4708       ּמי  I-ORG  \n",
       "3485      absolute  unspecified\\r\\r\\r\\n   הכנסייה        4730   הכנסייה  I-ORG  \n",
       "3557      absolute  unspecified\\r\\r\\r\\n      מסדר        4822       סדר  I-ORG  \n",
       "3571      absolute  unspecified\\r\\r\\r\\n       האו        4839       האו  I-ORG  \n",
       "3572      absolute  unspecified\\r\\r\\r\\n        ֹפ        4840         ֹ  I-ORG  \n",
       "3573      absolute  unspecified\\r\\r\\r\\n        ּו        4841        ּו  I-ORG  \n",
       "3574   unspecified  unspecified\\r\\r\\r\\n        ּס        4842        ּס  I-ORG  \n",
       "3575   unspecified         past\\r\\r\\r\\n        ֵא        4844        ֵא  I-ORG  \n",
       "4008      absolute  unspecified\\r\\r\\r\\n      מסדר        5468       סדר  I-ORG  \n",
       "4084      absolute  unspecified\\r\\r\\r\\n      ֶמפל        5572       ֶמף  I-ORG  \n",
       "4166      absolute  unspecified\\r\\r\\r\\n    הבונים        5683    הבונים  I-ORG  \n",
       "...            ...                  ...       ...         ...       ...    ...  \n",
       "44284     absolute  unspecified\\r\\r\\r\\n     לשכות       60537     לשכות  I-ORG  \n",
       "44678     absolute  unspecified\\r\\r\\r\\n     למשרד       61048      משרד  I-ORG  \n",
       "44679     absolute  unspecified\\r\\r\\r\\n     האוצר       61049     האוצר  I-ORG  \n",
       "44680     absolute  unspecified\\r\\r\\r\\n    ולמשרד       61050      משרד  I-ORG  \n",
       "44703     absolute  unspecified\\r\\r\\r\\n  בהתאחדות       61082   התאחדות  I-ORG  \n",
       "44849  unspecified  unspecified\\r\\r\\r\\n         -       61259         -  I-ORG  \n",
       "44850  unspecified  unspecified\\r\\r\\r\\n       סבן       61260       סבן  I-ORG  \n",
       "44851  unspecified  unspecified\\r\\r\\r\\n         -       61261         -  I-ORG  \n",
       "45214     absolute  unspecified\\r\\r\\r\\n      בבנק       61726       בנק  I-ORG  \n",
       "45704     absolute  unspecified\\r\\r\\r\\n     חיתום       62322     חיתום  I-ORG  \n",
       "45705  unspecified  unspecified\\r\\r\\r\\n  ורוסאריו       62323  ורוסאריו  I-ORG  \n",
       "45854    construct  unspecified\\r\\r\\r\\n    בקבוצת       62498     קבוצת  I-ORG  \n",
       "45859    construct  unspecified\\r\\r\\r\\n    חיפושי       62504    חיפושי  I-ORG  \n",
       "45970    construct  unspecified\\r\\r\\r\\n    מקבוצת       62637     קבוצת  I-ORG  \n",
       "46057  unspecified  unspecified\\r\\r\\r\\n         -       62743         -  I-ORG  \n",
       "46067  unspecified  unspecified\\r\\r\\r\\n         -       62755         -  I-ORG  \n",
       "46074     absolute  unspecified\\r\\r\\r\\n    התעשיה       62763    התעשיה  I-ORG  \n",
       "46195  unspecified  unspecified\\r\\r\\r\\n   Venture       62909   Venture  I-ORG  \n",
       "46196  unspecified  unspecified\\r\\r\\r\\n   Venture       62913   Venture  I-ORG  \n",
       "46197  unspecified  unspecified\\r\\r\\r\\n   Capital       62914   Capital  I-ORG  \n",
       "46349     absolute  unspecified\\r\\r\\r\\n    לאיחוד       63105     איחוד  I-ORG  \n",
       "46373     absolute  unspecified\\r\\r\\r\\n     במכון       63133      מכון  I-ORG  \n",
       "46410     absolute  unspecified\\r\\r\\r\\n    האיחוד       63178    האיחוד  I-ORG  \n",
       "46452     absolute  unspecified\\r\\r\\r\\n   מהאיחוד       63226    האיחוד  I-ORG  \n",
       "46482     absolute  unspecified\\r\\r\\r\\n    האיחוד       63263    האיחוד  I-ORG  \n",
       "46489     absolute  unspecified\\r\\r\\r\\n    באיחוד       63274     איחוד  I-ORG  \n",
       "46528     absolute  unspecified\\r\\r\\r\\n    לאיחוד       63324     איחוד  I-ORG  \n",
       "46567     absolute  unspecified\\r\\r\\r\\n     הלשכה       63374     הלשכה  I-ORG  \n",
       "46568     absolute  unspecified\\r\\r\\r\\n   הלאומית       63375   הלאומית  I-ORG  \n",
       "46574  unspecified  unspecified\\r\\r\\r\\n     בלהמן       63381      להמן  I-ORG  \n",
       "\n",
       "[477 rows x 12 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['BILUO'] == 'I-ORG']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Maybe we don't need all of the tokenization parts... ? What if - we keep only that parts where 'Word' == 'Token' ?\n",
    "Or - maybe we want only the token ? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Token</th>\n",
       "      <th>Word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>אבו-ג'וייד</td>\n",
       "      <td>אבו-ג'וייד</td>\n",
       "      <td>אבו-ג'וייד</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>הסכים</td>\n",
       "      <td>הסכים</td>\n",
       "      <td>הסכים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>יצא</td>\n",
       "      <td>ויצא</td>\n",
       "      <td>יצא</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>אימון</td>\n",
       "      <td>לאימונים</td>\n",
       "      <td>אימונים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>סוריה</td>\n",
       "      <td>בסוריה</td>\n",
       "      <td>סוריה</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>הוכשר</td>\n",
       "      <td>הוכשר</td>\n",
       "      <td>הוכשר</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>ירי</td>\n",
       "      <td>בירי</td>\n",
       "      <td>ירי</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210</th>\n",
       "      <td>נשק</td>\n",
       "      <td>נשק</td>\n",
       "      <td>נשק</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>כבד</td>\n",
       "      <td>וכבד</td>\n",
       "      <td>כבד</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213</th>\n",
       "      <td>ייצור</td>\n",
       "      <td>ייצור</td>\n",
       "      <td>ייצור</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214</th>\n",
       "      <td>מטען</td>\n",
       "      <td>מטענים</td>\n",
       "      <td>מטענים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>שימוש</td>\n",
       "      <td>ושימוש</td>\n",
       "      <td>שימוש</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>חומר</td>\n",
       "      <td>בחומרי</td>\n",
       "      <td>חומרי</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217</th>\n",
       "      <td>נפץ</td>\n",
       "      <td>נפץ</td>\n",
       "      <td>נפץ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>סיום</td>\n",
       "      <td>סיום</td>\n",
       "      <td>סיום</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>אימון</td>\n",
       "      <td>האימונים</td>\n",
       "      <td>האימונים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>קיבל</td>\n",
       "      <td>קיבל</td>\n",
       "      <td>קיבל</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>אבו-ג'וייד</td>\n",
       "      <td>אבו-ג'וייד</td>\n",
       "      <td>אבו-ג'וייד</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223</th>\n",
       "      <td>תשלום</td>\n",
       "      <td>תשלום</td>\n",
       "      <td>תשלום</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>224</th>\n",
       "      <td>שב</td>\n",
       "      <td>ושב</td>\n",
       "      <td>שב</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225</th>\n",
       "      <td>ירדן</td>\n",
       "      <td>לירדן</td>\n",
       "      <td>ירדן</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226</th>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>227</th>\n",
       "      <td>הובהר</td>\n",
       "      <td>שהובהר</td>\n",
       "      <td>הובהר</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228</th>\n",
       "      <td>הוכשר</td>\n",
       "      <td>שהוכשר</td>\n",
       "      <td>הוכשר</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>סייע</td>\n",
       "      <td>לסייע</td>\n",
       "      <td>לסייע</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>מאבק</td>\n",
       "      <td>למאבק</td>\n",
       "      <td>מאבק</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>פלסטיני</td>\n",
       "      <td>הפלשתיני</td>\n",
       "      <td>הפלשתיני</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "      <td>,</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234</th>\n",
       "      <td>החליט</td>\n",
       "      <td>החליט</td>\n",
       "      <td>החליט</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>אבו-ג'וייד</td>\n",
       "      <td>אבו-ג'וייד</td>\n",
       "      <td>אבו-ג'וייד</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>יצא</td>\n",
       "      <td>לצאת</td>\n",
       "      <td>לצאת</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>עיראק</td>\n",
       "      <td>לעיראק</td>\n",
       "      <td>עיראק</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>סייע</td>\n",
       "      <td>לסייע</td>\n",
       "      <td>לסייע</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>לחימה</td>\n",
       "      <td>בלחימה</td>\n",
       "      <td>לחימה</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>אמריקני</td>\n",
       "      <td>האמריקנים</td>\n",
       "      <td>האמריקנים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>שבוע</td>\n",
       "      <td>כשבועיים</td>\n",
       "      <td>שבועיים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>לחם</td>\n",
       "      <td>לחם</td>\n",
       "      <td>לחם</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>אבו-גוויד</td>\n",
       "      <td>אבו-גוויד</td>\n",
       "      <td>אבו-גוויד</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>מורד</td>\n",
       "      <td>המורדים</td>\n",
       "      <td>המורדים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>עיראקי</td>\n",
       "      <td>העיראקיים</td>\n",
       "      <td>העיראקיים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>נפצע</td>\n",
       "      <td>נפצע</td>\n",
       "      <td>נפצע</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>מירי</td>\n",
       "      <td>מירי</td>\n",
       "      <td>מירי</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249</th>\n",
       "      <td>חזר</td>\n",
       "      <td>וחזר</td>\n",
       "      <td>חזר</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Lemma       Token        Word\n",
       "200           .           .           .\n",
       "201  אבו-ג'וייד  אבו-ג'וייד  אבו-ג'וייד\n",
       "202       הסכים       הסכים       הסכים\n",
       "203           ,           ,           ,\n",
       "204         יצא        ויצא         יצא\n",
       "205       אימון    לאימונים     אימונים\n",
       "206       סוריה      בסוריה       סוריה\n",
       "207           ,           ,           ,\n",
       "208       הוכשר       הוכשר       הוכשר\n",
       "209         ירי        בירי         ירי\n",
       "210         נשק         נשק         נשק\n",
       "211         כבד        וכבד         כבד\n",
       "212           ,           ,           ,\n",
       "213       ייצור       ייצור       ייצור\n",
       "214        מטען      מטענים      מטענים\n",
       "215       שימוש      ושימוש       שימוש\n",
       "216        חומר      בחומרי       חומרי\n",
       "217         נפץ         נפץ         נפץ\n",
       "218           .           .           .\n",
       "219        סיום        סיום        סיום\n",
       "220       אימון    האימונים    האימונים\n",
       "221        קיבל        קיבל        קיבל\n",
       "222  אבו-ג'וייד  אבו-ג'וייד  אבו-ג'וייד\n",
       "223       תשלום       תשלום       תשלום\n",
       "224          שב         ושב          שב\n",
       "225        ירדן       לירדן        ירדן\n",
       "226           .           .           .\n",
       "227       הובהר      שהובהר       הובהר\n",
       "228       הוכשר      שהוכשר       הוכשר\n",
       "229        סייע       לסייע       לסייע\n",
       "230        מאבק       למאבק        מאבק\n",
       "231     פלסטיני    הפלשתיני    הפלשתיני\n",
       "232           ,           ,           ,\n",
       "233           -           -           -\n",
       "234       החליט       החליט       החליט\n",
       "235  אבו-ג'וייד  אבו-ג'וייד  אבו-ג'וייד\n",
       "236         יצא        לצאת        לצאת\n",
       "237       עיראק      לעיראק       עיראק\n",
       "238        סייע       לסייע       לסייע\n",
       "239       לחימה      בלחימה       לחימה\n",
       "240     אמריקני   האמריקנים   האמריקנים\n",
       "241           .           .           .\n",
       "242        שבוע    כשבועיים     שבועיים\n",
       "243         לחם         לחם         לחם\n",
       "244   אבו-גוויד   אבו-גוויד   אבו-גוויד\n",
       "245        מורד     המורדים     המורדים\n",
       "246      עיראקי   העיראקיים   העיראקיים\n",
       "247        נפצע        נפצע        נפצע\n",
       "248        מירי        מירי        מירי\n",
       "249         חזר        וחזר         חזר"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['Lemma', 'Token', 'Word']][200:250]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Maybe we won't to filter all of the 'single char' words?\n",
    "\n",
    "And maybe - we want the word embedding on the lemma instead of token? \n",
    "\n",
    "Let's try - step by step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Token</th>\n",
       "      <th>Word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>781</th>\n",
       "      <td>יווני</td>\n",
       "      <td>היוונים</td>\n",
       "      <td>היוונים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3426</th>\n",
       "      <td>ן</td>\n",
       "      <td>צופן</td>\n",
       "      <td>ן</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3427</th>\n",
       "      <td>דה</td>\n",
       "      <td>דה</td>\n",
       "      <td>דה</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3503</th>\n",
       "      <td>ן</td>\n",
       "      <td>צופן</td>\n",
       "      <td>ן</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3504</th>\n",
       "      <td>דה</td>\n",
       "      <td>דה</td>\n",
       "      <td>דה</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3666</th>\n",
       "      <td>סוד</td>\n",
       "      <td>הסודות</td>\n",
       "      <td>הסודות</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3667</th>\n",
       "      <td>צפה</td>\n",
       "      <td>צופן</td>\n",
       "      <td>צוף</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3668</th>\n",
       "      <td>ן</td>\n",
       "      <td>צופן</td>\n",
       "      <td>ן</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3669</th>\n",
       "      <td>דה</td>\n",
       "      <td>דה</td>\n",
       "      <td>דה</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3708</th>\n",
       "      <td>ברית</td>\n",
       "      <td>בברית</td>\n",
       "      <td>ברית</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3778</th>\n",
       "      <td>גילה</td>\n",
       "      <td>מגילת</td>\n",
       "      <td>גילת</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3800</th>\n",
       "      <td>ברית</td>\n",
       "      <td>בברית</td>\n",
       "      <td>ברית</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3937</th>\n",
       "      <td>סוד</td>\n",
       "      <td>הסודות</td>\n",
       "      <td>הסודות</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3938</th>\n",
       "      <td>צפה</td>\n",
       "      <td>צופן</td>\n",
       "      <td>צוף</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3939</th>\n",
       "      <td>ן</td>\n",
       "      <td>צופן</td>\n",
       "      <td>ן</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3940</th>\n",
       "      <td>דה</td>\n",
       "      <td>דה</td>\n",
       "      <td>דה</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4419</th>\n",
       "      <td>דם</td>\n",
       "      <td>הדם</td>\n",
       "      <td>הדם</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4420</th>\n",
       "      <td>קדוש</td>\n",
       "      <td>הקדוש</td>\n",
       "      <td>הקדוש</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4421</th>\n",
       "      <td>גביע</td>\n",
       "      <td>והגביע</td>\n",
       "      <td>הגביע</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4598</th>\n",
       "      <td>ן</td>\n",
       "      <td>צופן</td>\n",
       "      <td>ן</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4599</th>\n",
       "      <td>דה</td>\n",
       "      <td>דה</td>\n",
       "      <td>דה</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7864</th>\n",
       "      <td>יריד</td>\n",
       "      <td>ביריד</td>\n",
       "      <td>יריד</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7865</th>\n",
       "      <td>בינלאומי</td>\n",
       "      <td>הבינלאומי</td>\n",
       "      <td>הבינלאומי</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7866</th>\n",
       "      <td>שנתי</td>\n",
       "      <td>השנתי</td>\n",
       "      <td>השנתי</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7867</th>\n",
       "      <td>ספר</td>\n",
       "      <td>ספרי</td>\n",
       "      <td>ספרי</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8202</th>\n",
       "      <td>רחוב</td>\n",
       "      <td>ברחוב</td>\n",
       "      <td>רחוב</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8675</th>\n",
       "      <td>שמש</td>\n",
       "      <td>השמש</td>\n",
       "      <td>השמש</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8874</th>\n",
       "      <td>תל</td>\n",
       "      <td>התל</td>\n",
       "      <td>התל</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9774</th>\n",
       "      <td>התחיל</td>\n",
       "      <td>מתחילה</td>\n",
       "      <td>מתחילה</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9930</th>\n",
       "      <td>מצביע</td>\n",
       "      <td>המצביע</td>\n",
       "      <td>המצביע</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10182</th>\n",
       "      <td>אקונומיק</td>\n",
       "      <td>אקונומיק</td>\n",
       "      <td>אקונומיק</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10183</th>\n",
       "      <td>ריווי</td>\n",
       "      <td>ריוויו</td>\n",
       "      <td>ריווי</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10662</th>\n",
       "      <td>הפורדים</td>\n",
       "      <td>הפורדים</td>\n",
       "      <td>הפורדים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10756</th>\n",
       "      <td>מלחמה</td>\n",
       "      <td>כשהמלחמה</td>\n",
       "      <td>המלחמה</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10939</th>\n",
       "      <td>ברוקר</td>\n",
       "      <td>הברוקרים</td>\n",
       "      <td>הברוקרים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10940</th>\n",
       "      <td>רעיון</td>\n",
       "      <td>הרעיונות</td>\n",
       "      <td>הרעיונות</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10941</th>\n",
       "      <td>:</td>\n",
       "      <td>:</td>\n",
       "      <td>:</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10942</th>\n",
       "      <td>צוות</td>\n",
       "      <td>צוותות</td>\n",
       "      <td>צוותות</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10943</th>\n",
       "      <td>חשיבה</td>\n",
       "      <td>חשיבה</td>\n",
       "      <td>חשיבה</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10944</th>\n",
       "      <td>עלייה</td>\n",
       "      <td>ועלייתה</td>\n",
       "      <td>עליית</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10945</th>\n",
       "      <td>עלייה</td>\n",
       "      <td>עלית</td>\n",
       "      <td>עלית</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10946</th>\n",
       "      <td>מדיניות</td>\n",
       "      <td>מדיניות</td>\n",
       "      <td>מדיניות</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10981</th>\n",
       "      <td>מלחמה</td>\n",
       "      <td>למלחמה</td>\n",
       "      <td>מלחמה</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17079</th>\n",
       "      <td>אירי</td>\n",
       "      <td>האירים</td>\n",
       "      <td>האירים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17401</th>\n",
       "      <td>מלחמה</td>\n",
       "      <td>במלחמת</td>\n",
       "      <td>מלחמת</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17733</th>\n",
       "      <td>מלחמה</td>\n",
       "      <td>במלחמת</td>\n",
       "      <td>מלחמת</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17811</th>\n",
       "      <td>מרקחת</td>\n",
       "      <td>מרקחת</td>\n",
       "      <td>מרקחת</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17812</th>\n",
       "      <td>גטו</td>\n",
       "      <td>בגטו</td>\n",
       "      <td>גטו</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18365</th>\n",
       "      <td>שש</td>\n",
       "      <td>ששת</td>\n",
       "      <td>ששת</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18390</th>\n",
       "      <td>קרב</td>\n",
       "      <td>הקרב</td>\n",
       "      <td>הקרב</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Lemma      Token       Word\n",
       "781       יווני    היוונים    היוונים\n",
       "3426          ן       צופן          ן\n",
       "3427         דה         דה         דה\n",
       "3503          ן       צופן          ן\n",
       "3504         דה         דה         דה\n",
       "3666        סוד     הסודות     הסודות\n",
       "3667        צפה       צופן        צוף\n",
       "3668          ן       צופן          ן\n",
       "3669         דה         דה         דה\n",
       "3708       ברית      בברית       ברית\n",
       "3778       גילה      מגילת       גילת\n",
       "3800       ברית      בברית       ברית\n",
       "3937        סוד     הסודות     הסודות\n",
       "3938        צפה       צופן        צוף\n",
       "3939          ן       צופן          ן\n",
       "3940         דה         דה         דה\n",
       "4419         דם        הדם        הדם\n",
       "4420       קדוש      הקדוש      הקדוש\n",
       "4421       גביע     והגביע      הגביע\n",
       "4598          ן       צופן          ן\n",
       "4599         דה         דה         דה\n",
       "7864       יריד      ביריד       יריד\n",
       "7865   בינלאומי  הבינלאומי  הבינלאומי\n",
       "7866       שנתי      השנתי      השנתי\n",
       "7867        ספר       ספרי       ספרי\n",
       "8202       רחוב      ברחוב       רחוב\n",
       "8675        שמש       השמש       השמש\n",
       "8874         תל        התל        התל\n",
       "9774      התחיל     מתחילה     מתחילה\n",
       "9930      מצביע     המצביע     המצביע\n",
       "10182  אקונומיק   אקונומיק   אקונומיק\n",
       "10183     ריווי     ריוויו      ריווי\n",
       "10662   הפורדים    הפורדים    הפורדים\n",
       "10756     מלחמה   כשהמלחמה     המלחמה\n",
       "10939     ברוקר   הברוקרים   הברוקרים\n",
       "10940     רעיון   הרעיונות   הרעיונות\n",
       "10941         :          :          :\n",
       "10942      צוות     צוותות     צוותות\n",
       "10943     חשיבה      חשיבה      חשיבה\n",
       "10944     עלייה    ועלייתה      עליית\n",
       "10945     עלייה       עלית       עלית\n",
       "10946   מדיניות    מדיניות    מדיניות\n",
       "10981     מלחמה     למלחמה      מלחמה\n",
       "17079      אירי     האירים     האירים\n",
       "17401     מלחמה     במלחמת      מלחמת\n",
       "17733     מלחמה     במלחמת      מלחמת\n",
       "17811     מרקחת      מרקחת      מרקחת\n",
       "17812       גטו       בגטו        גטו\n",
       "18365        שש        ששת        ששת\n",
       "18390       קרב       הקרב       הקרב"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i_misc_data[['Lemma', 'Token', 'Word']].head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bio</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Number</th>\n",
       "      <th>Person</th>\n",
       "      <th>Pos</th>\n",
       "      <th>Status</th>\n",
       "      <th>Tense</th>\n",
       "      <th>Token</th>\n",
       "      <th>TokenOrder</th>\n",
       "      <th>Word</th>\n",
       "      <th>BILUO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>781</th>\n",
       "      <td>I_MISC__AFF</td>\n",
       "      <td>masculine</td>\n",
       "      <td>יווני</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>adjective</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>היוונים</td>\n",
       "      <td>1149</td>\n",
       "      <td>היוונים</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Bio     Gender  Lemma  Number       Person        Pos    Status  \\\n",
       "781  I_MISC__AFF  masculine  יווני  plural  unspecified  adjective  absolute   \n",
       "\n",
       "                   Tense    Token  TokenOrder     Word   BILUO  \n",
       "781  unspecified\\r\\r\\r\\n  היוונים        1149  היוונים  I-MISC  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i_misc_data[i_misc_data['Word'] == 'היוונים']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bio</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Number</th>\n",
       "      <th>Person</th>\n",
       "      <th>Pos</th>\n",
       "      <th>Status</th>\n",
       "      <th>Tense</th>\n",
       "      <th>Token</th>\n",
       "      <th>TokenOrder</th>\n",
       "      <th>Word</th>\n",
       "      <th>BILUO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>781</th>\n",
       "      <td>I_MISC__AFF</td>\n",
       "      <td>masculine</td>\n",
       "      <td>יווני</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>adjective</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>היוונים</td>\n",
       "      <td>1149</td>\n",
       "      <td>היוונים</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3427</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>דה</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>דה</td>\n",
       "      <td>4659</td>\n",
       "      <td>דה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3504</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>דה</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>דה</td>\n",
       "      <td>4754</td>\n",
       "      <td>דה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3666</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>סוד</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הסודות</td>\n",
       "      <td>4971</td>\n",
       "      <td>הסודות</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3667</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>צפה</td>\n",
       "      <td>singular</td>\n",
       "      <td>any</td>\n",
       "      <td>participle</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>צופן</td>\n",
       "      <td>4973</td>\n",
       "      <td>צוף</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3669</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>דה</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>דה</td>\n",
       "      <td>4974</td>\n",
       "      <td>דה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3708</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ברית</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בברית</td>\n",
       "      <td>5025</td>\n",
       "      <td>ברית</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3778</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>גילה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>מגילת</td>\n",
       "      <td>5123</td>\n",
       "      <td>גילת</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3800</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ברית</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בברית</td>\n",
       "      <td>5155</td>\n",
       "      <td>ברית</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3937</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>סוד</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הסודות</td>\n",
       "      <td>5361</td>\n",
       "      <td>הסודות</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3938</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>צפה</td>\n",
       "      <td>singular</td>\n",
       "      <td>any</td>\n",
       "      <td>participle</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>צופן</td>\n",
       "      <td>5363</td>\n",
       "      <td>צוף</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3940</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>דה</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>דה</td>\n",
       "      <td>5364</td>\n",
       "      <td>דה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4419</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>דם</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הדם</td>\n",
       "      <td>6039</td>\n",
       "      <td>הדם</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4420</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>קדוש</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הקדוש</td>\n",
       "      <td>6040</td>\n",
       "      <td>הקדוש</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4421</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>גביע</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>והגביע</td>\n",
       "      <td>6041</td>\n",
       "      <td>הגביע</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4599</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>דה</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>דה</td>\n",
       "      <td>6294</td>\n",
       "      <td>דה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7864</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>יריד</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ביריד</td>\n",
       "      <td>10982</td>\n",
       "      <td>יריד</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7865</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>בינלאומי</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>adjective</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הבינלאומי</td>\n",
       "      <td>10983</td>\n",
       "      <td>הבינלאומי</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7866</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>שנתי</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>adjective</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>השנתי</td>\n",
       "      <td>10984</td>\n",
       "      <td>השנתי</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7867</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>ספר</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ספרי</td>\n",
       "      <td>10986</td>\n",
       "      <td>ספרי</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8202</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>רחוב</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ברחוב</td>\n",
       "      <td>11404</td>\n",
       "      <td>רחוב</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8675</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>שמש</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>השמש</td>\n",
       "      <td>12029</td>\n",
       "      <td>השמש</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8874</th>\n",
       "      <td>I_MISC__AFF</td>\n",
       "      <td>masculine</td>\n",
       "      <td>תל</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>התל</td>\n",
       "      <td>12296</td>\n",
       "      <td>התל</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9774</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>התחיל</td>\n",
       "      <td>singular</td>\n",
       "      <td>any</td>\n",
       "      <td>participle</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>מתחילה</td>\n",
       "      <td>13527</td>\n",
       "      <td>מתחילה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9930</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>מצביע</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>המצביע</td>\n",
       "      <td>13730</td>\n",
       "      <td>המצביע</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10182</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>אקונומיק</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>adjective</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>אקונומיק</td>\n",
       "      <td>14081</td>\n",
       "      <td>אקונומיק</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10183</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>ריווי</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ריוויו</td>\n",
       "      <td>14082</td>\n",
       "      <td>ריווי</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10662</th>\n",
       "      <td>I_MISC__AFF</td>\n",
       "      <td>masculine</td>\n",
       "      <td>הפורדים</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הפורדים</td>\n",
       "      <td>14745</td>\n",
       "      <td>הפורדים</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10756</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>מלחמה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>כשהמלחמה</td>\n",
       "      <td>14874</td>\n",
       "      <td>המלחמה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10939</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>ברוקר</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הברוקרים</td>\n",
       "      <td>15106</td>\n",
       "      <td>הברוקרים</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23746</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>קדם</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הקדם</td>\n",
       "      <td>33233</td>\n",
       "      <td>הקדם</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24133</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>נוח</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>adjective</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>נוחות</td>\n",
       "      <td>33800</td>\n",
       "      <td>נוחות</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24134</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>יהודי</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ליהודים</td>\n",
       "      <td>33801</td>\n",
       "      <td>יהודים</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24314</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>קאמף</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>קאמף</td>\n",
       "      <td>34064</td>\n",
       "      <td>קאמף</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25771</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ליגה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בליגה</td>\n",
       "      <td>36048</td>\n",
       "      <td>ליגה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27722</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>מלחמה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>במלחמת</td>\n",
       "      <td>38723</td>\n",
       "      <td>מלחמת</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28309</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>גביע</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בגביע</td>\n",
       "      <td>39530</td>\n",
       "      <td>גביע</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28345</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>גביע</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בגביע</td>\n",
       "      <td>39579</td>\n",
       "      <td>גביע</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28365</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>גביע</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בגביע</td>\n",
       "      <td>39602</td>\n",
       "      <td>גביע</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28366</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>התאחדות</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ההתאחדות</td>\n",
       "      <td>39603</td>\n",
       "      <td>ההתאחדות</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28376</th>\n",
       "      <td>I_MISC__AFF</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ישראלי</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>adjective</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הישראלית</td>\n",
       "      <td>39619</td>\n",
       "      <td>הישראלית</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28377</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>אליפות</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>לאליפות</td>\n",
       "      <td>39620</td>\n",
       "      <td>אליפות</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28378</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>הרמה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בהרמת</td>\n",
       "      <td>39622</td>\n",
       "      <td>הרמת</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28657</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>אולימפיאדה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>באולימפיאדת</td>\n",
       "      <td>40009</td>\n",
       "      <td>אולימפיאדת</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29063</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>מלחמה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>למלחמת</td>\n",
       "      <td>40591</td>\n",
       "      <td>מלחמת</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29225</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>מלחמה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>המלחמה</td>\n",
       "      <td>40813</td>\n",
       "      <td>המלחמה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30072</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>masculine</td>\n",
       "      <td>יומן</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ביומן</td>\n",
       "      <td>41909</td>\n",
       "      <td>יומן</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30944</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>אמנה</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>באמנות</td>\n",
       "      <td>43071</td>\n",
       "      <td>אמנות</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30945</th>\n",
       "      <td>I_MISC__ENT</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>האג</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>האג</td>\n",
       "      <td>43072</td>\n",
       "      <td>האג</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33262</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>שש</td>\n",
       "      <td>singular</td>\n",
       "      <td>any</td>\n",
       "      <td>participle</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ששת</td>\n",
       "      <td>46235</td>\n",
       "      <td>ששת</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36658</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>אליפות</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>באליפות</td>\n",
       "      <td>50799</td>\n",
       "      <td>אליפות</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36773</th>\n",
       "      <td>I_MISC__AFF</td>\n",
       "      <td>feminine</td>\n",
       "      <td>ישראלי</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>adjective</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>הישראלית</td>\n",
       "      <td>50943</td>\n",
       "      <td>הישראלית</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36774</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>אולימפיאדה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>לאולימפיאדת</td>\n",
       "      <td>50944</td>\n",
       "      <td>אולימפיאדת</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37280</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>אליפות</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>באליפות</td>\n",
       "      <td>51601</td>\n",
       "      <td>אליפות</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37281</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>אירופה</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>אירופה</td>\n",
       "      <td>51602</td>\n",
       "      <td>אירופה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37282</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>קפיצה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>בקפיצה</td>\n",
       "      <td>51603</td>\n",
       "      <td>קפיצה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37546</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>אינתיפאדה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>האינתיפאדה</td>\n",
       "      <td>51961</td>\n",
       "      <td>האינתיפאדה</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38338</th>\n",
       "      <td>I_MISC__AFF</td>\n",
       "      <td>masculine</td>\n",
       "      <td>יהודי</td>\n",
       "      <td>plural</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>absolute</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>שיהודים</td>\n",
       "      <td>53013</td>\n",
       "      <td>יהודים</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39200</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>feminine</td>\n",
       "      <td>מלחמה</td>\n",
       "      <td>singular</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>noun</td>\n",
       "      <td>construct</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>במלחמת</td>\n",
       "      <td>54162</td>\n",
       "      <td>מלחמת</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39201</th>\n",
       "      <td>I_MISC_EVENT</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>שש</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>properName</td>\n",
       "      <td>unspecified</td>\n",
       "      <td>unspecified\\r\\r\\r\\n</td>\n",
       "      <td>ששת</td>\n",
       "      <td>54163</td>\n",
       "      <td>ששת</td>\n",
       "      <td>I-MISC</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                Bio       Gender       Lemma       Number       Person  \\\n",
       "781     I_MISC__AFF    masculine       יווני       plural  unspecified   \n",
       "3427    I_MISC__ENT  unspecified          דה  unspecified  unspecified   \n",
       "3504    I_MISC__ENT  unspecified          דה  unspecified  unspecified   \n",
       "3666    I_MISC__ENT    masculine         סוד       plural  unspecified   \n",
       "3667    I_MISC__ENT    masculine         צפה     singular          any   \n",
       "3669    I_MISC__ENT  unspecified          דה  unspecified  unspecified   \n",
       "3708    I_MISC__ENT     feminine        ברית     singular  unspecified   \n",
       "3778    I_MISC__ENT     feminine        גילה     singular  unspecified   \n",
       "3800    I_MISC__ENT     feminine        ברית     singular  unspecified   \n",
       "3937    I_MISC__ENT    masculine         סוד       plural  unspecified   \n",
       "3938    I_MISC__ENT    masculine         צפה     singular          any   \n",
       "3940    I_MISC__ENT  unspecified          דה  unspecified  unspecified   \n",
       "4419    I_MISC__ENT    masculine          דם     singular  unspecified   \n",
       "4420    I_MISC__ENT    masculine        קדוש     singular  unspecified   \n",
       "4421    I_MISC__ENT    masculine        גביע     singular  unspecified   \n",
       "4599    I_MISC__ENT  unspecified          דה  unspecified  unspecified   \n",
       "7864   I_MISC_EVENT    masculine        יריד     singular  unspecified   \n",
       "7865   I_MISC_EVENT    masculine    בינלאומי     singular  unspecified   \n",
       "7866   I_MISC_EVENT    masculine        שנתי     singular  unspecified   \n",
       "7867   I_MISC_EVENT    masculine         ספר       plural  unspecified   \n",
       "8202    I_MISC__ENT    masculine        רחוב     singular  unspecified   \n",
       "8675    I_MISC__ENT     feminine         שמש     singular  unspecified   \n",
       "8874    I_MISC__AFF    masculine          תל     singular  unspecified   \n",
       "9774    I_MISC__ENT     feminine       התחיל     singular          any   \n",
       "9930    I_MISC__ENT    masculine       מצביע     singular  unspecified   \n",
       "10182   I_MISC__ENT     feminine    אקונומיק     singular  unspecified   \n",
       "10183   I_MISC__ENT    masculine       ריווי     singular  unspecified   \n",
       "10662   I_MISC__AFF    masculine     הפורדים       plural  unspecified   \n",
       "10756  I_MISC_EVENT     feminine       מלחמה     singular  unspecified   \n",
       "10939   I_MISC__ENT    masculine       ברוקר       plural  unspecified   \n",
       "...             ...          ...         ...          ...          ...   \n",
       "23746  I_MISC_EVENT    masculine         קדם     singular  unspecified   \n",
       "24133   I_MISC__ENT     feminine         נוח       plural  unspecified   \n",
       "24134   I_MISC__ENT    masculine       יהודי       plural  unspecified   \n",
       "24314   I_MISC__ENT  unspecified        קאמף  unspecified  unspecified   \n",
       "25771  I_MISC_EVENT     feminine        ליגה     singular  unspecified   \n",
       "27722  I_MISC_EVENT     feminine       מלחמה     singular  unspecified   \n",
       "28309  I_MISC_EVENT    masculine        גביע     singular  unspecified   \n",
       "28345  I_MISC_EVENT    masculine        גביע     singular  unspecified   \n",
       "28365  I_MISC_EVENT    masculine        גביע     singular  unspecified   \n",
       "28366  I_MISC_EVENT     feminine     התאחדות     singular  unspecified   \n",
       "28376   I_MISC__AFF     feminine      ישראלי     singular  unspecified   \n",
       "28377  I_MISC_EVENT     feminine      אליפות     singular  unspecified   \n",
       "28378  I_MISC_EVENT     feminine        הרמה     singular  unspecified   \n",
       "28657  I_MISC_EVENT     feminine  אולימפיאדה     singular  unspecified   \n",
       "29063  I_MISC_EVENT     feminine       מלחמה     singular  unspecified   \n",
       "29225  I_MISC_EVENT     feminine       מלחמה     singular  unspecified   \n",
       "30072   I_MISC__ENT    masculine        יומן     singular  unspecified   \n",
       "30944   I_MISC__ENT     feminine        אמנה       plural  unspecified   \n",
       "30945   I_MISC__ENT  unspecified         האג  unspecified  unspecified   \n",
       "33262  I_MISC_EVENT     feminine          שש     singular          any   \n",
       "36658  I_MISC_EVENT     feminine      אליפות     singular  unspecified   \n",
       "36773   I_MISC__AFF     feminine      ישראלי     singular  unspecified   \n",
       "36774  I_MISC_EVENT     feminine  אולימפיאדה     singular  unspecified   \n",
       "37280  I_MISC_EVENT     feminine      אליפות     singular  unspecified   \n",
       "37281  I_MISC_EVENT  unspecified      אירופה  unspecified  unspecified   \n",
       "37282  I_MISC_EVENT     feminine       קפיצה     singular  unspecified   \n",
       "37546  I_MISC_EVENT     feminine   אינתיפאדה     singular  unspecified   \n",
       "38338   I_MISC__AFF    masculine       יהודי       plural  unspecified   \n",
       "39200  I_MISC_EVENT     feminine       מלחמה     singular  unspecified   \n",
       "39201  I_MISC_EVENT  unspecified          שש  unspecified  unspecified   \n",
       "\n",
       "              Pos       Status                Tense        Token  TokenOrder  \\\n",
       "781     adjective     absolute  unspecified\\r\\r\\r\\n      היוונים        1149   \n",
       "3427   properName  unspecified  unspecified\\r\\r\\r\\n           דה        4659   \n",
       "3504   properName  unspecified  unspecified\\r\\r\\r\\n           דה        4754   \n",
       "3666         noun     absolute  unspecified\\r\\r\\r\\n       הסודות        4971   \n",
       "3667   participle  unspecified  unspecified\\r\\r\\r\\n         צופן        4973   \n",
       "3669   properName  unspecified  unspecified\\r\\r\\r\\n           דה        4974   \n",
       "3708         noun     absolute  unspecified\\r\\r\\r\\n        בברית        5025   \n",
       "3778         noun    construct  unspecified\\r\\r\\r\\n        מגילת        5123   \n",
       "3800         noun     absolute  unspecified\\r\\r\\r\\n        בברית        5155   \n",
       "3937         noun     absolute  unspecified\\r\\r\\r\\n       הסודות        5361   \n",
       "3938   participle  unspecified  unspecified\\r\\r\\r\\n         צופן        5363   \n",
       "3940   properName  unspecified  unspecified\\r\\r\\r\\n           דה        5364   \n",
       "4419         noun     absolute  unspecified\\r\\r\\r\\n          הדם        6039   \n",
       "4420         noun     absolute  unspecified\\r\\r\\r\\n        הקדוש        6040   \n",
       "4421         noun     absolute  unspecified\\r\\r\\r\\n       והגביע        6041   \n",
       "4599   properName  unspecified  unspecified\\r\\r\\r\\n           דה        6294   \n",
       "7864         noun     absolute  unspecified\\r\\r\\r\\n        ביריד       10982   \n",
       "7865    adjective     absolute  unspecified\\r\\r\\r\\n    הבינלאומי       10983   \n",
       "7866    adjective     absolute  unspecified\\r\\r\\r\\n        השנתי       10984   \n",
       "7867         noun    construct  unspecified\\r\\r\\r\\n         ספרי       10986   \n",
       "8202         noun     absolute  unspecified\\r\\r\\r\\n        ברחוב       11404   \n",
       "8675         noun     absolute  unspecified\\r\\r\\r\\n         השמש       12029   \n",
       "8874         noun     absolute  unspecified\\r\\r\\r\\n          התל       12296   \n",
       "9774   participle     absolute  unspecified\\r\\r\\r\\n       מתחילה       13527   \n",
       "9930         noun     absolute  unspecified\\r\\r\\r\\n       המצביע       13730   \n",
       "10182   adjective     absolute  unspecified\\r\\r\\r\\n     אקונומיק       14081   \n",
       "10183        noun     absolute  unspecified\\r\\r\\r\\n       ריוויו       14082   \n",
       "10662        noun     absolute  unspecified\\r\\r\\r\\n      הפורדים       14745   \n",
       "10756        noun     absolute  unspecified\\r\\r\\r\\n     כשהמלחמה       14874   \n",
       "10939        noun     absolute  unspecified\\r\\r\\r\\n     הברוקרים       15106   \n",
       "...           ...          ...                  ...          ...         ...   \n",
       "23746        noun     absolute  unspecified\\r\\r\\r\\n         הקדם       33233   \n",
       "24133   adjective     absolute  unspecified\\r\\r\\r\\n        נוחות       33800   \n",
       "24134        noun     absolute  unspecified\\r\\r\\r\\n      ליהודים       33801   \n",
       "24314  properName  unspecified  unspecified\\r\\r\\r\\n         קאמף       34064   \n",
       "25771        noun     absolute  unspecified\\r\\r\\r\\n        בליגה       36048   \n",
       "27722        noun    construct  unspecified\\r\\r\\r\\n       במלחמת       38723   \n",
       "28309        noun     absolute  unspecified\\r\\r\\r\\n        בגביע       39530   \n",
       "28345        noun     absolute  unspecified\\r\\r\\r\\n        בגביע       39579   \n",
       "28365        noun     absolute  unspecified\\r\\r\\r\\n        בגביע       39602   \n",
       "28366        noun     absolute  unspecified\\r\\r\\r\\n     ההתאחדות       39603   \n",
       "28376   adjective     absolute  unspecified\\r\\r\\r\\n     הישראלית       39619   \n",
       "28377        noun     absolute  unspecified\\r\\r\\r\\n      לאליפות       39620   \n",
       "28378        noun    construct  unspecified\\r\\r\\r\\n        בהרמת       39622   \n",
       "28657        noun    construct  unspecified\\r\\r\\r\\n  באולימפיאדת       40009   \n",
       "29063        noun    construct  unspecified\\r\\r\\r\\n       למלחמת       40591   \n",
       "29225        noun     absolute  unspecified\\r\\r\\r\\n       המלחמה       40813   \n",
       "30072        noun     absolute  unspecified\\r\\r\\r\\n        ביומן       41909   \n",
       "30944        noun     absolute  unspecified\\r\\r\\r\\n       באמנות       43071   \n",
       "30945  properName  unspecified  unspecified\\r\\r\\r\\n          האג       43072   \n",
       "33262  participle    construct  unspecified\\r\\r\\r\\n          ששת       46235   \n",
       "36658        noun     absolute  unspecified\\r\\r\\r\\n      באליפות       50799   \n",
       "36773   adjective     absolute  unspecified\\r\\r\\r\\n     הישראלית       50943   \n",
       "36774        noun    construct  unspecified\\r\\r\\r\\n  לאולימפיאדת       50944   \n",
       "37280        noun     absolute  unspecified\\r\\r\\r\\n      באליפות       51601   \n",
       "37281  properName  unspecified  unspecified\\r\\r\\r\\n       אירופה       51602   \n",
       "37282        noun     absolute  unspecified\\r\\r\\r\\n       בקפיצה       51603   \n",
       "37546        noun     absolute  unspecified\\r\\r\\r\\n   האינתיפאדה       51961   \n",
       "38338        noun     absolute  unspecified\\r\\r\\r\\n      שיהודים       53013   \n",
       "39200        noun    construct  unspecified\\r\\r\\r\\n       במלחמת       54162   \n",
       "39201  properName  unspecified  unspecified\\r\\r\\r\\n          ששת       54163   \n",
       "\n",
       "             Word   BILUO  \n",
       "781       היוונים  I-MISC  \n",
       "3427           דה  I-MISC  \n",
       "3504           דה  I-MISC  \n",
       "3666       הסודות  I-MISC  \n",
       "3667          צוף  I-MISC  \n",
       "3669           דה  I-MISC  \n",
       "3708         ברית  I-MISC  \n",
       "3778         גילת  I-MISC  \n",
       "3800         ברית  I-MISC  \n",
       "3937       הסודות  I-MISC  \n",
       "3938          צוף  I-MISC  \n",
       "3940           דה  I-MISC  \n",
       "4419          הדם  I-MISC  \n",
       "4420        הקדוש  I-MISC  \n",
       "4421        הגביע  I-MISC  \n",
       "4599           דה  I-MISC  \n",
       "7864         יריד  I-MISC  \n",
       "7865    הבינלאומי  I-MISC  \n",
       "7866        השנתי  I-MISC  \n",
       "7867         ספרי  I-MISC  \n",
       "8202         רחוב  I-MISC  \n",
       "8675         השמש  I-MISC  \n",
       "8874          התל  I-MISC  \n",
       "9774       מתחילה  I-MISC  \n",
       "9930       המצביע  I-MISC  \n",
       "10182    אקונומיק  I-MISC  \n",
       "10183       ריווי  I-MISC  \n",
       "10662     הפורדים  I-MISC  \n",
       "10756      המלחמה  I-MISC  \n",
       "10939    הברוקרים  I-MISC  \n",
       "...           ...     ...  \n",
       "23746        הקדם  I-MISC  \n",
       "24133       נוחות  I-MISC  \n",
       "24134      יהודים  I-MISC  \n",
       "24314        קאמף  I-MISC  \n",
       "25771        ליגה  I-MISC  \n",
       "27722       מלחמת  I-MISC  \n",
       "28309        גביע  I-MISC  \n",
       "28345        גביע  I-MISC  \n",
       "28365        גביע  I-MISC  \n",
       "28366    ההתאחדות  I-MISC  \n",
       "28376    הישראלית  I-MISC  \n",
       "28377      אליפות  I-MISC  \n",
       "28378        הרמת  I-MISC  \n",
       "28657  אולימפיאדת  I-MISC  \n",
       "29063       מלחמת  I-MISC  \n",
       "29225      המלחמה  I-MISC  \n",
       "30072        יומן  I-MISC  \n",
       "30944       אמנות  I-MISC  \n",
       "30945         האג  I-MISC  \n",
       "33262         ששת  I-MISC  \n",
       "36658      אליפות  I-MISC  \n",
       "36773    הישראלית  I-MISC  \n",
       "36774  אולימפיאדת  I-MISC  \n",
       "37280      אליפות  I-MISC  \n",
       "37281      אירופה  I-MISC  \n",
       "37282       קפיצה  I-MISC  \n",
       "37546  האינתיפאדה  I-MISC  \n",
       "38338      יהודים  I-MISC  \n",
       "39200       מלחמת  I-MISC  \n",
       "39201         ששת  I-MISC  \n",
       "\n",
       "[100 rows x 12 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i_misc_data[i_misc_data['Word'].apply(lambda x: len(x) > 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df = df[df['Word'].apply(lambda x: len(x) > 1)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In addition, let's add another word vec to number? of all ones?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lemma</th>\n",
       "      <th>Token</th>\n",
       "      <th>Word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>DOCSTART</td>\n",
       "      <td>DOCSTART</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ישראלי</td>\n",
       "      <td>ישראלים</td>\n",
       "      <td>ישראלים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>אוכל</td>\n",
       "      <td>אוכלים</td>\n",
       "      <td>אוכלים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>שליש</td>\n",
       "      <td>שליש</td>\n",
       "      <td>שליש</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>נהנה</td>\n",
       "      <td>נהנים</td>\n",
       "      <td>נהנים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>אחוז</td>\n",
       "      <td>אחוז</td>\n",
       "      <td>אחוז</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>אוכל</td>\n",
       "      <td>אוכלים</td>\n",
       "      <td>אוכלים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ערב</td>\n",
       "      <td>בערב</td>\n",
       "      <td>ערב</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>אילו</td>\n",
       "      <td>ואילו</td>\n",
       "      <td>אילו</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>אוכל</td>\n",
       "      <td>אוכלים</td>\n",
       "      <td>אוכלים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>צהריים</td>\n",
       "      <td>בצהריים</td>\n",
       "      <td>צהריים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>נתן</td>\n",
       "      <td>נתון</td>\n",
       "      <td>נתון</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>עניין</td>\n",
       "      <td>מעניין</td>\n",
       "      <td>עניין</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>נוסף</td>\n",
       "      <td>נוסף</td>\n",
       "      <td>נוסף</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>איש</td>\n",
       "      <td>הנשים</td>\n",
       "      <td>הנשים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>ישראלי</td>\n",
       "      <td>הישראליות</td>\n",
       "      <td>הישראליות</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>אכל</td>\n",
       "      <td>אוכלות</td>\n",
       "      <td>אוכלות</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>גבר</td>\n",
       "      <td>מהגברים</td>\n",
       "      <td>הגברים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>ישראלי</td>\n",
       "      <td>מהישראליות</td>\n",
       "      <td>הישראליות</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>צרך</td>\n",
       "      <td>צורכות</td>\n",
       "      <td>צורכות</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>דגן</td>\n",
       "      <td>דגני</td>\n",
       "      <td>דגני</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>סדיר</td>\n",
       "      <td>סדיר</td>\n",
       "      <td>סדיר</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>לעומת</td>\n",
       "      <td>לעומת</td>\n",
       "      <td>לעומת</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "      <td>###NUMBER###</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>גבר</td>\n",
       "      <td>מהגברים</td>\n",
       "      <td>הגברים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>עלה</td>\n",
       "      <td>עולה</td>\n",
       "      <td>עול</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>סקר</td>\n",
       "      <td>מסקר</td>\n",
       "      <td>סקר</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>ערך</td>\n",
       "      <td>שערכה</td>\n",
       "      <td>ערכה</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>חבר</td>\n",
       "      <td>חברת</td>\n",
       "      <td>חברת</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>קלוגס</td>\n",
       "      <td>קלוגס</td>\n",
       "      <td>קלוגס</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>מיכון</td>\n",
       "      <td>מכון</td>\n",
       "      <td>מכון</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>חקר</td>\n",
       "      <td>חקר</td>\n",
       "      <td>חקר</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>רייטינג</td>\n",
       "      <td>רייטינג</td>\n",
       "      <td>רייטינג</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>נתון</td>\n",
       "      <td>מהנתונים</td>\n",
       "      <td>הנתונים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>עלה</td>\n",
       "      <td>עולה</td>\n",
       "      <td>עול</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>קבוצה</td>\n",
       "      <td>קבוצת</td>\n",
       "      <td>קבוצת</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>גיל</td>\n",
       "      <td>הגיל</td>\n",
       "      <td>הגיל</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>אכל</td>\n",
       "      <td>שאוכלת</td>\n",
       "      <td>אוכלת</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "      <td>קורנפלקס</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>###NUMEXP###</td>\n",
       "      <td>###NUMEXP###</td>\n",
       "      <td>###NUMEXP###</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>שליש</td>\n",
       "      <td>שליש</td>\n",
       "      <td>שליש</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>צורך</td>\n",
       "      <td>צורכים</td>\n",
       "      <td>צורכים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>סוג</td>\n",
       "      <td>סוג</td>\n",
       "      <td>סוג</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>קבע</td>\n",
       "      <td>קבוע</td>\n",
       "      <td>קבוע</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Lemma         Token          Word\n",
       "0       DOCSTART      DOCSTART      DOCSTART\n",
       "1         ישראלי       ישראלים       ישראלים\n",
       "2           אוכל        אוכלים        אוכלים\n",
       "3       קורנפלקס      קורנפלקס      קורנפלקס\n",
       "5           שליש          שליש          שליש\n",
       "6           נהנה         נהנים         נהנים\n",
       "8   ###NUMBER###  ###NUMBER###  ###NUMBER###\n",
       "9           אחוז          אחוז          אחוז\n",
       "10          אוכל        אוכלים        אוכלים\n",
       "11           ערב          בערב           ערב\n",
       "13          אילו         ואילו          אילו\n",
       "14  ###NUMBER###  ###NUMBER###  ###NUMBER###\n",
       "16          אוכל        אוכלים        אוכלים\n",
       "17      קורנפלקס      קורנפלקס      קורנפלקס\n",
       "18        צהריים       בצהריים        צהריים\n",
       "20           נתן          נתון          נתון\n",
       "21         עניין        מעניין         עניין\n",
       "22          נוסף          נוסף          נוסף\n",
       "24           איש         הנשים         הנשים\n",
       "25        ישראלי     הישראליות     הישראליות\n",
       "26           אכל        אוכלות        אוכלות\n",
       "27      קורנפלקס      קורנפלקס      קורנפלקס\n",
       "28           גבר       מהגברים        הגברים\n",
       "30  ###NUMBER###  ###NUMBER###  ###NUMBER###\n",
       "32        ישראלי    מהישראליות     הישראליות\n",
       "33           צרך        צורכות        צורכות\n",
       "34           דגן          דגני          דגני\n",
       "35          סדיר          סדיר          סדיר\n",
       "37         לעומת         לעומת         לעומת\n",
       "38  ###NUMBER###  ###NUMBER###  ###NUMBER###\n",
       "40           גבר       מהגברים        הגברים\n",
       "42           עלה          עולה           עול\n",
       "43           סקר          מסקר           סקר\n",
       "44           ערך         שערכה          ערכה\n",
       "45           חבר          חברת          חברת\n",
       "47         קלוגס         קלוגס         קלוגס\n",
       "49         מיכון          מכון          מכון\n",
       "51           חקר           חקר           חקר\n",
       "52       רייטינג       רייטינג       רייטינג\n",
       "55          נתון      מהנתונים       הנתונים\n",
       "56           עלה          עולה           עול\n",
       "57         קבוצה         קבוצת         קבוצת\n",
       "58           גיל          הגיל          הגיל\n",
       "59           אכל        שאוכלת         אוכלת\n",
       "60      קורנפלקס      קורנפלקס      קורנפלקס\n",
       "62  ###NUMEXP###  ###NUMEXP###  ###NUMEXP###\n",
       "64          שליש          שליש          שליש\n",
       "65          צורך        צורכים        צורכים\n",
       "66           סוג           סוג           סוג\n",
       "67           קבע          קבוע          קבוע"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_df[['Lemma', 'Token', 'Word']].head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "data2_path = 'resources' + os.sep + 'train_data.csv'    \n",
    "df2 = pd.read_csv(data2_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4624"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2['next_wordvec_0'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "NLPConda36",
   "language": "python",
   "name": "nlpconda36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}