fetch_datasets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyodbc\n",
    "import csv\n",
    "import xml.etree.cElementTree as et\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMOTIONS = ['joy', 'anger', 'sadness', 'fear']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fetching ISEAR Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "MDB = \"./ISEAR_0/isear_databank/isear_databank.mdb\"\n",
    "DRV = \"{Microsoft Access Driver (*.mdb, *.accdb)}\"\n",
    "PWD = \"pw\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "conn = pyodbc.connect('DRIVER={}; DBQ={}; PWD={}'.format(DRV, MDB, PWD))\n",
    "cursor = conn.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>CITY</th>\n",
       "      <th>COUN</th>\n",
       "      <th>SUBJ</th>\n",
       "      <th>SEX</th>\n",
       "      <th>AGE</th>\n",
       "      <th>RELI</th>\n",
       "      <th>PRAC</th>\n",
       "      <th>FOCC</th>\n",
       "      <th>MOCC</th>\n",
       "      <th>...</th>\n",
       "      <th>SELF</th>\n",
       "      <th>RELA</th>\n",
       "      <th>VERBAL</th>\n",
       "      <th>NEUTRO</th>\n",
       "      <th>Field1</th>\n",
       "      <th>Field3</th>\n",
       "      <th>Field2</th>\n",
       "      <th>MYKEY</th>\n",
       "      <th>SIT</th>\n",
       "      <th>STATE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11001.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>joy</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>0110011</td>\n",
       "      <td>During the period of falling in love, each tim...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11001.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>fear</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0110012</td>\n",
       "      <td>When I was involved in a traffic accident.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11001.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>anger</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0110013</td>\n",
       "      <td>When I was driving home after  several days of...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11001.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>sadness</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0110014</td>\n",
       "      <td>When I lost the person who meant the most to me.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11001.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>disgust</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0110015</td>\n",
       "      <td>The time I knocked a deer down - the sight of ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7661</th>\n",
       "      <td>331062.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>anger</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3310623</td>\n",
       "      <td>Two years back someone invited me to be the tu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7662</th>\n",
       "      <td>331062.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>sadness</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3310624</td>\n",
       "      <td>I had taken the responsibility to do something...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7663</th>\n",
       "      <td>331062.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>disgust</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3310625</td>\n",
       "      <td>I was at home and I heard a loud sound of spit...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7664</th>\n",
       "      <td>331062.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>shame</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3310626</td>\n",
       "      <td>I did not do the homework that the teacher had...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7665</th>\n",
       "      <td>331062.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>guilt</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3310627</td>\n",
       "      <td>I had shouted at my younger brother and he was...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7666 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            ID  CITY  COUN  SUBJ  SEX   AGE  RELI  PRAC  FOCC  MOCC  ...  \\\n",
       "0      11001.0   1.0   1.0   1.0  1.0  33.0   1.0   2.0   6.0   1.0  ...   \n",
       "1      11001.0   1.0   1.0   1.0  1.0  33.0   1.0   2.0   6.0   1.0  ...   \n",
       "2      11001.0   1.0   1.0   1.0  1.0  33.0   1.0   2.0   6.0   1.0  ...   \n",
       "3      11001.0   1.0   1.0   1.0  1.0  33.0   1.0   2.0   6.0   1.0  ...   \n",
       "4      11001.0   1.0   1.0   1.0  1.0  33.0   1.0   2.0   6.0   1.0  ...   \n",
       "...        ...   ...   ...   ...  ...   ...   ...   ...   ...   ...  ...   \n",
       "7661  331062.0   1.0  33.0  62.0  2.0  21.0   2.0   1.0   7.0   7.0  ...   \n",
       "7662  331062.0   1.0  33.0  62.0  2.0  21.0   2.0   1.0   7.0   7.0  ...   \n",
       "7663  331062.0   1.0  33.0  62.0  2.0  21.0   2.0   1.0   7.0   7.0  ...   \n",
       "7664  331062.0   1.0  33.0  62.0  2.0  21.0   2.0   1.0   7.0   7.0  ...   \n",
       "7665  331062.0   1.0  33.0  62.0  2.0  21.0   2.0   1.0   7.0   7.0  ...   \n",
       "\n",
       "      SELF  RELA  VERBAL  NEUTRO   Field1  Field3  Field2    MYKEY  \\\n",
       "0      3.0   3.0     2.0     0.0      joy       4       3  0110011   \n",
       "1      2.0   2.0     0.0     0.0     fear       3       2  0110012   \n",
       "2      2.0   1.0     0.0     0.0    anger       1       3  0110013   \n",
       "3      1.0   1.0     0.0     2.0  sadness       4       4  0110014   \n",
       "4      0.0   2.0     0.0     0.0  disgust       4       4  0110015   \n",
       "...    ...   ...     ...     ...      ...     ...     ...      ...   \n",
       "7661   2.0   2.0     3.0     0.0    anger       1       2  3310623   \n",
       "7662   2.0   0.0     1.0     1.0  sadness       4       3  3310624   \n",
       "7663   2.0   2.0     0.0     0.0  disgust       1       2  3310625   \n",
       "7664   2.0   0.0     2.0     0.0    shame       1       3  3310626   \n",
       "7665   1.0   2.0     1.0     1.0    guilt       1       2  3310627   \n",
       "\n",
       "                                                    SIT  STATE  \n",
       "0     During the period of falling in love, each tim...      1  \n",
       "1            When I was involved in a traffic accident.      1  \n",
       "2     When I was driving home after  several days of...      1  \n",
       "3     When I lost the person who meant the most to me.       1  \n",
       "4     The time I knocked a deer down - the sight of ...      1  \n",
       "...                                                 ...    ...  \n",
       "7661  Two years back someone invited me to be the tu...      1  \n",
       "7662  I had taken the responsibility to do something...      1  \n",
       "7663  I was at home and I heard a loud sound of spit...      1  \n",
       "7664  I did not do the homework that the teacher had...      1  \n",
       "7665  I had shouted at my younger brother and he was...      1  \n",
       "\n",
       "[7666 rows x 42 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "QUERY = 'SELECT * FROM data;' \n",
    "rows = cursor.execute(QUERY).fetchall()\n",
    "ISEAR_df = pd.DataFrame([tuple(row) for row in rows], columns=[column[0] for column in cursor.description])\n",
    "cursor.close()\n",
    "conn.close()\n",
    "ISEAR_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ISEAR_labels</th>\n",
       "      <th>ISEAR_sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>joy</td>\n",
       "      <td>During the period of falling in love, each tim...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fear</td>\n",
       "      <td>When I was involved in a traffic accident.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>anger</td>\n",
       "      <td>When I was driving home after  several days of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sadness</td>\n",
       "      <td>When I lost the person who meant the most to me.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>disgust</td>\n",
       "      <td>The time I knocked a deer down - the sight of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7661</th>\n",
       "      <td>anger</td>\n",
       "      <td>Two years back someone invited me to be the tu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7662</th>\n",
       "      <td>sadness</td>\n",
       "      <td>I had taken the responsibility to do something...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7663</th>\n",
       "      <td>disgust</td>\n",
       "      <td>I was at home and I heard a loud sound of spit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7664</th>\n",
       "      <td>shame</td>\n",
       "      <td>I did not do the homework that the teacher had...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7665</th>\n",
       "      <td>guilt</td>\n",
       "      <td>I had shouted at my younger brother and he was...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7666 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     ISEAR_labels                                    ISEAR_sentences\n",
       "0             joy  During the period of falling in love, each tim...\n",
       "1            fear         When I was involved in a traffic accident.\n",
       "2           anger  When I was driving home after  several days of...\n",
       "3         sadness  When I lost the person who meant the most to me. \n",
       "4         disgust  The time I knocked a deer down - the sight of ...\n",
       "...           ...                                                ...\n",
       "7661        anger  Two years back someone invited me to be the tu...\n",
       "7662      sadness  I had taken the responsibility to do something...\n",
       "7663      disgust  I was at home and I heard a loud sound of spit...\n",
       "7664        shame  I did not do the homework that the teacher had...\n",
       "7665        guilt  I had shouted at my younger brother and he was...\n",
       "\n",
       "[7666 rows x 2 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ISEAR_df = ISEAR_df.iloc[:, [36, 40]]\n",
    "ISEAR_df.columns = ['ISEAR_labels', 'ISEAR_sentences']\n",
    "ISEAR_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ISEAR_labels\n",
       "anger      1096\n",
       "sadness    1096\n",
       "disgust    1096\n",
       "shame      1096\n",
       "fear       1095\n",
       "joy        1094\n",
       "guilt      1093\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ISEAR_df['ISEAR_labels'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ISEAR_labels</th>\n",
       "      <th>ISEAR_sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>joy</td>\n",
       "      <td>During the period of falling in love, each tim...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fear</td>\n",
       "      <td>When I was involved in a traffic accident.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>anger</td>\n",
       "      <td>When I was driving home after  several days of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sadness</td>\n",
       "      <td>When I lost the person who meant the most to me.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>joy</td>\n",
       "      <td>When I got a letter offering me the Summer job...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4376</th>\n",
       "      <td>sadness</td>\n",
       "      <td>This issue worried me rather than saddening me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4377</th>\n",
       "      <td>joy</td>\n",
       "      <td>I received a letter from a distant friend.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4378</th>\n",
       "      <td>fear</td>\n",
       "      <td>My parents were out and I was the eldest at ho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4379</th>\n",
       "      <td>anger</td>\n",
       "      <td>Two years back someone invited me to be the tu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4380</th>\n",
       "      <td>sadness</td>\n",
       "      <td>I had taken the responsibility to do something...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4381 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     ISEAR_labels                                    ISEAR_sentences\n",
       "0             joy  During the period of falling in love, each tim...\n",
       "1            fear         When I was involved in a traffic accident.\n",
       "2           anger  When I was driving home after  several days of...\n",
       "3         sadness  When I lost the person who meant the most to me. \n",
       "4             joy  When I got a letter offering me the Summer job...\n",
       "...           ...                                                ...\n",
       "4376      sadness  This issue worried me rather than saddening me...\n",
       "4377          joy         I received a letter from a distant friend.\n",
       "4378         fear  My parents were out and I was the eldest at ho...\n",
       "4379        anger  Two years back someone invited me to be the tu...\n",
       "4380      sadness  I had taken the responsibility to do something...\n",
       "\n",
       "[4381 rows x 2 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ISEAR_df_filtered = ISEAR_df[ISEAR_df.ISEAR_labels.isin(EMOTIONS)].reset_index(drop=True)\n",
    "ISEAR_df_filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ISEAR_CSV = \"./datasets_csv/ISEAR_dataset.csv\"\n",
    "ISEAR_df_filtered.to_csv(ISEAR_CSV)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fetching AffectiveText Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "XML1 = \"./AffectiveText.Semeval.2007/AffectiveText.test/affectivetext_test.xml\"\n",
    "XML2 = \"./AffectiveText.Semeval.2007/AffectiveText.trial/affectivetext_trial.xml\"\n",
    "LABELS1 = \"./AffectiveText.Semeval.2007/AffectiveText.test/affectivetext_test.emotions.gold\"\n",
    "LABELS2 = \"./AffectiveText.Semeval.2007/AffectiveText.trial/affectivetext_trial.emotions.gold\"\n",
    "VALENCE1 = \"./AffectiveText.Semeval.2007/AffectiveText.test/affectivetext_test.valence.gold\"\n",
    "VALENCE2 = \"./AffectiveText.Semeval.2007/AffectiveText.trial/affectivetext_trial.valence.gold\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree=et.parse(XML2)\n",
    "root=tree.getroot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Mortar assault leaves at least 18 dead',\n",
       " 'Goal delight for Sheva',\n",
       " 'Nigeria hostage feared dead is freed',\n",
       " 'Bombers kill shoppers',\n",
       " 'Vegetables, not fruit, slow brain decline',\n",
       " 'PM: Havana deal a good experiment',\n",
       " 'Kate is marrying Doherty',\n",
       " 'NASA revisiting life on Mars question',\n",
       " 'Happy birthday, iPod',\n",
       " 'Alonso would be happy to retire with three titles',\n",
       " \"Madonna's New Tot 'Happy at Home' in London\",\n",
       " \"Nicole Kidman asks dad to help stop husband's drinking\",\n",
       " 'United Finds Good Connection in Win',\n",
       " \"'Runway': Making Good Without Making Nice\",\n",
       " \"We were 'arrogant and stupid' over Iraq, says US diplomat\",\n",
       " 'Bad reasons to be good',\n",
       " \"Madonna's new baby's daddy didn't realize adoption was `for good'\",\n",
       " \"Women in their 60s 'are perfectly good mothers'\",\n",
       " \"We're a pretty kind 'bully'\",\n",
       " \"Moderate drinking reduces men's heart attack risk\",\n",
       " 'Tom Cruise and Katie Holmes set wedding date',\n",
       " 'Kidnapped AP photographer freed in Gaza',\n",
       " 'Bush Insists Troops Stay in Iraq, Predicts Midterm Victory',\n",
       " 'Hurricane Paul Weakens To Tropical Storm',\n",
       " 'UK announces immigration restrictions',\n",
       " 'Three found alive from missing Russian ship -report',\n",
       " 'Poison Pill to Swallow: Hawks Hurting After Loss to Vikes',\n",
       " 'Injured Marathon Winner Leaves Hospital',\n",
       " 'Hacker unlocks Apple music download protection',\n",
       " 'Spanish journalist seized in Gaza',\n",
       " 'Sudan urges UN to replace its special envoy',\n",
       " 'Resolution approved for international games',\n",
       " 'Pair of Satellites Will Document Sun in 3-D',\n",
       " 'Martian Life Could Have Evaded Detection by Viking Landers',\n",
       " 'The sweet tune of an anniversary',\n",
       " 'Did Viking Probes Miss Life on Mars?',\n",
       " 'Diplomat acknowledges US arrogance in Iraq',\n",
       " 'More Human Remains Found at Ground Zero',\n",
       " 'How to Prevent Iraq From Getting Even Worse',\n",
       " 'Tensions as Hungary marks uprising',\n",
       " 'Stanford study warns of Internet addiction',\n",
       " 'No evidence of ice reserves on the moon',\n",
       " 'A Picture and A Thousand Words',\n",
       " 'Parachutist dies at bridge-jump festival',\n",
       " 'US Troops Killed In October In Iraq',\n",
       " 'Chefs not counting calories, study finds',\n",
       " 'Deaths linked to flu vaccine',\n",
       " 'Perfect lives lose their focus',\n",
       " 'Bread-Heavy Diet Linked to Kidney Cancer',\n",
       " 'Beating poverty in a small way',\n",
       " 'Israel insists on Lebanon overflights',\n",
       " 'Britain to restrict immigrants from new EU members',\n",
       " 'Confusion Reigns In the Expanding Digital World',\n",
       " 'Game on! London exhibition celebrates the history of video games',\n",
       " 'Making peace from victory over poverty',\n",
       " 'Marine killed in fighting west of Baghdad',\n",
       " 'Scientists create first cloaking device',\n",
       " 'Amateur rocket scientists reach for space',\n",
       " 'Confusion Reigns In the Expanding Digital World',\n",
       " 'Deaths linked to flu vaccine',\n",
       " 'Israel admits using phosphorous shells in Lebanon',\n",
       " '4th person dies after receiving flu shot',\n",
       " 'Police and shoppers targeted in growing Iraq violence',\n",
       " 'Scientists discover miracle in the depths',\n",
       " \"Doctors Seeing Patients Who Think They're Internet Addicted\",\n",
       " 'China launches two satellites into orbit',\n",
       " 'Vegetables May Boost Brain Power in Older Adults',\n",
       " 'Hurricane Paul nears Category 3 status',\n",
       " 'Russian ship sinks, 18-member crew missing',\n",
       " \"Iran's president urges higher birth rate\",\n",
       " 'Apple set for massive market gains',\n",
       " 'Tumor Types May Explain Survival Rates for Cancer',\n",
       " 'Routine mission turns into hunt for missing comrade',\n",
       " 'Bad weather slows S.Korean search for Russian ship',\n",
       " 'Tailors and housewives busy as Muslims celebrate Eid',\n",
       " 'Active peace brings the promise of development',\n",
       " 'Abbas condemns Israeli massacre in northern Gaza Strip',\n",
       " 'Bangladesh ferry sinks, 15 dead',\n",
       " 'Bucs find a new kick in life',\n",
       " 'Google to unveil election mashup',\n",
       " 'Police warn of child exploitation online',\n",
       " 'Breast Cancer Awareness is Important',\n",
       " 'Hungary police tackle violent protestors',\n",
       " 'Three drivers had reason to be proud in Brazil',\n",
       " 'Celebrities Protest Malibu Gas Facility',\n",
       " 'Lebo: A life lived on the edge',\n",
       " \"Presto! He's Batman, then magic man\",\n",
       " 'Heavy use of mobile phones can make men infertile, scientists say',\n",
       " 'Russian ship sinks in Pacific',\n",
       " 'China Successfully Launches Two Satellites',\n",
       " 'Prestige has magic touch',\n",
       " 'Lies, damned lies, and divorce',\n",
       " 'Manning inspires Giants to victory in Dallas',\n",
       " 'Riot warning for France suburbs',\n",
       " 'Full recovery expected for marathon winner',\n",
       " 'Sony apologises for global battery recall',\n",
       " 'Questions on credit card safety',\n",
       " 'Budapest calm after night of violent protests',\n",
       " 'Vegetables May Boost Brain Power in Older Adults',\n",
       " 'Report criticises US press freedoms',\n",
       " 'French suburb riots could resume',\n",
       " 'Photographer kidnapping condemned',\n",
       " 'Photographer kidnapped in Gaza',\n",
       " 'Baseball: Rogers enjoys his muddied streak',\n",
       " 'Cultural revolutionary, the iPod is five',\n",
       " 'Life on Mars? Maybe We Missed It',\n",
       " 'Three found alive from missing Russian ship -report',\n",
       " \"Moderate drinking reduces men's heart attack risk\",\n",
       " 'The sweet tune of an anniversary',\n",
       " 'Inconvenient truths',\n",
       " 'Dating difficult for Harry Potter',\n",
       " 'Students lose their rite of passage',\n",
       " 'Putin vows to tackle illegal immigration',\n",
       " \"Ottawa must redefine 'terror'\",\n",
       " 'Britain to Limit Workers from Romania, Bulgaria',\n",
       " 'Top General Optimistic on Iraq',\n",
       " 'Kim Osorio Wins Sexual Harrassment Suit Against The Source',\n",
       " 'Silly and substantial',\n",
       " 'Official laughs for Borat',\n",
       " 'Speaker Hastert testifies before panel probing sex scandal',\n",
       " 'On Educating Sexes Separately',\n",
       " 'Tom Cruise and Katie Holmes set wedding date',\n",
       " 'Martian Life Could Have Evaded Detection by Viking Landers',\n",
       " 'Ukraine gas deal marred by fears',\n",
       " 'Marathon victory comes at heavy cost',\n",
       " 'College tuition continues climbing',\n",
       " 'German paper shows soldiers desecrating skull',\n",
       " 'War is close, Ethiopia tells Somali rebels',\n",
       " 'Advantage Cardinals with Suppan in Game 4',\n",
       " 'NASA spacecraft to measure Sun',\n",
       " 'Sony Hates Europeans, Will Prevent The Importing of PS3',\n",
       " 'Flu Vaccine Appears Safe for Young Children',\n",
       " 'Google To Launch Yet Another Search Tool',\n",
       " \"Tropical Storm Paul's Outer Bands Hit Mexico's Baja California\",\n",
       " 'Israeli Bomblets Plague Lebanon',\n",
       " 'New Indonesia Calamity, a Mud Bath, Is Man-Made',\n",
       " 'U.S. Weighs Sanctions Against North Korea',\n",
       " 'Rice, in Baghdad, Insists That Iraqis Are ',\n",
       " 'Senator Says U.S. Should Rethink Iraq Strategy',\n",
       " 'German Chief Forges Accord on Financing Health Care',\n",
       " 'Turkish Writers Say Efforts to Stifle Speech May Backfire',\n",
       " 'Kurdish Lawmaker Killed in an Attack in Baghdad',\n",
       " 'Anti-U.S. Attack Videos Spread on the Internet',\n",
       " 'Women Face Greatest Threat of Violence at Home, Study Finds',\n",
       " 'Appeal by Chinese Researcher for Times May Soon Be Heard',\n",
       " 'Brazil Air Force Cites Faults and Confusion in Fatal Crash',\n",
       " 'A Mix of Oil and Environmentalism',\n",
       " 'Immigrants and French Reach Deal in Standoff',\n",
       " 'Genghis Khan Beer? Mongolia Grimaces',\n",
       " 'Dead Bachelors in Remote China Still Find Wives',\n",
       " 'Managua Journal: Hold the Mojito and Margarita, Nicaragua Has el Macu',\n",
       " 'Paris Journal: Smoking No Longer Tres Chic in France',\n",
       " \"An Old Letter Casts Doubts on Iran's Goal for Uranium\",\n",
       " 'U.S. Warns North Koreans About Nuclear Test',\n",
       " 'Global Sludge Ends in Tragedy for Ivory Coast',\n",
       " 'Equipment on Plane in Brazil Collision May Have Been Faulty',\n",
       " 'New Indonesia Calamity, a Mud Bath, Is Man-Made',\n",
       " 'Iraqi Journalists Add Laws to List of Dangers',\n",
       " 'U.N. Force Is Treading Lightly on Lebanese Soil',\n",
       " 'Cleric Said to Lose Reins Over Part of Iraqi Militia',\n",
       " \"Citing Spy Agencies' Study, Democrats Seek Iraq Changes\",\n",
       " 'Opera Canceled Over a Depiction of Muhammad',\n",
       " \"Military Officials Add to U.S. Criticism of Iraq's Government\",\n",
       " 'Teacher in Hiding After Attack on Islam Stirs Threats',\n",
       " 'Combative Bush Releases Parts of Terror Study',\n",
       " 'Growing Unarmed Battalion in Qaeda Army Is Using Internet to Get the Message Out',\n",
       " 'Iraqi Linked to Sunni Bloc Is Held in Plot, Military Says',\n",
       " 'Seduced by Snacks? No, Not You',\n",
       " 'Friends for Life: An Emerging Biology of Emotional Healing',\n",
       " \"Alzheimer's Drugs Offer No Help, Study Finds\",\n",
       " 'Across Europe, Worries on Islam Spread to Center',\n",
       " 'Cooking Defines Sephardic Jews at Sukkot',\n",
       " 'Fitness: Bodies in Motion, Clean and Sober',\n",
       " \"In God's Name: Religion-Based Tax Breaks: Housing to Paychecks to Books\",\n",
       " 'Yankee Dies in Plane Crash, Official Says',\n",
       " 'An Elephant Crackup?',\n",
       " 'With YouTube, Grad Student Hits Jackpot Again',\n",
       " 'Dow flirts with record highs again',\n",
       " 'Tropical Storm Isaac forms in Atlantic',\n",
       " 'New Iraq terror tape calls for abducting foreigners',\n",
       " \"'Deep regret' for ex-HP exec\",\n",
       " \"'Opportunity' knocks on geologist's dream crater\",\n",
       " 'Squirrel jumps boy in park; rabies suspected',\n",
       " 'Girl left comatose after dental visit dies',\n",
       " 'Porn star candidate campaigns on campus',\n",
       " 'School ends football season after 0-4 start',\n",
       " 'Charlie Sheen to become highest-paid sitcom star',\n",
       " \"Jon Stewart mocks benefits of CNN's I\",\n",
       " 'Ashton Kutcher tells secret to Ryan Seacrest',\n",
       " 'Nails-on-chalkboard study earns Ig',\n",
       " \"Gunman 'fine' before shooting\",\n",
       " \"Marijuana may stave off Alzheimer's\",\n",
       " 'Kentucky seeks alcohol huffing ban',\n",
       " 'Toddler died from tainted spinach',\n",
       " \"Judge dismisses Karr's porn case\",\n",
       " '10 die in 22-hour Kashmir standoff',\n",
       " 'Slain Amish girls laid to rest',\n",
       " '5,000 flee plant fire',\n",
       " 'Cher auction fetches double pre-sale estimates',\n",
       " \"Stars scuffle on 'Grey's Anatomy'\",\n",
       " 'Probe under way in Yankee crash',\n",
       " 'Amish schoolhouse torn down',\n",
       " \"Warner will not run in '08\",\n",
       " 'Reid land deal under scrutiny',\n",
       " 'Flooding washes out road to Valdez, Alaska',\n",
       " 'Briton admits U.S. bomb plot role',\n",
       " 'Commentary: Minutemen have a right to be idiotic',\n",
       " 'Madonna files adoption papers',\n",
       " 'Gunmen kill 9 at Iraqi TV station',\n",
       " 'Hastert sorry for scandal but not quitting',\n",
       " 'Ex-JonBenet suspect: Child porn case dropped ',\n",
       " 'Waste plant fire forces 5,000 to evacuate ',\n",
       " 'Toddler died from E. coli tainted spinach ',\n",
       " 'Disturbing finds in child sex raids ',\n",
       " 'Magazine lists women who had abortions ',\n",
       " \"Pot smokers may avoid Alzheimer's, study says \",\n",
       " \"Student starts white men's club on campus \",\n",
       " 'Kentucky wants to ban alcohol inhalers ',\n",
       " 'Peter Pan flies into chapter two ',\n",
       " 'Teen repellent inventor wins infamous prize ',\n",
       " \"Cher's dress, car, bed have new owners\",\n",
       " 'Dow hits new record, eyes 12,000 ',\n",
       " 'Ex-Virginia gov not running for president ',\n",
       " \"Yankee pitcher's pilot log found \",\n",
       " \"Dem leader's $1 million land deal under scrutiny \",\n",
       " 'Dad locked up son, watched on TV, police say ',\n",
       " 'Navarrette: Minutemen have a right to be idiotic ',\n",
       " 'Pit bulls jump miniature pony, leave her bloody ',\n",
       " 'Alaska town cut off from rest of state by flood ',\n",
       " 'Trailblazing trio protects City by the Bay ',\n",
       " 'Madonna files adoption papers in Malawi',\n",
       " \"'Dr. McDreamy' choked in 'Grey's' scuffle \",\n",
       " '$10,000 poetry prize awarded ',\n",
       " 'Las Vegas honour for tiger pair ',\n",
       " 'UK characters boosting box office ',\n",
       " 'Cher items snapped up at auction ',\n",
       " \"Griffiths scorns Withnail 'play' \",\n",
       " 'Lily Allen wins web music award ',\n",
       " 'Costner appeals in casino wrangle',\n",
       " 'Beyonce copyright claim rejected ',\n",
       " 'Cohn gunman jailed for 36 years ',\n",
       " \"Melua's deep sea gig sets record \",\n",
       " 'Sri Lanka clashes kill 129 troops ',\n",
       " 'Gunmen kill 11 in Iraq TV raid ',\n",
       " 'Italy drops Sicily bridge plans ',\n",
       " 'Amish killings school demolished ',\n",
       " 'Pamuk wins Nobel Literature prize',\n",
       " 'Madonna adoption bid approved ',\n",
       " 'Man admits UK-US terror bomb plot',\n",
       " 'Who are you calling fat?']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AFFECTIVETEXT_sentences = []\n",
    "for instance in root.iter('instance'):\n",
    "    AFFECTIVETEXT_sentences.append(instance.text)\n",
    "\n",
    "AFFECTIVETEXT_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}