override_patches_sentiment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fa19729e",
   "metadata": {},
   "source": [
    "### Use this notebook to reproduce results of Tables 3, 4, 5 (except Yelp Colloquial since that uses the feature based patcher). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "09f0b16e",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28ab20c9",
   "metadata": {},
   "source": [
    "#### Load in the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "59c586ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of T5ForConditionalGenerationMultipleHeads were not initialized from the model checkpoint at t5-large and are newly initialized: ['encoder.embed_tokens.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "primary mode: exp_applies_predictor\n",
      "splicing parts from pretrained model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of T5ForConditionalGeneration were not initialized from the model checkpoint at t5-large and are newly initialized: ['encoder.embed_tokens.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from eval_utils import load_model\n",
    "\n",
    "path_name = '/u/scr/smurty/LanguageExplanations/trained_models/t5-large-sst-override_exp-final'\n",
    "model_obj = load_model(path_name, primary_mode='exp_applies_predictor')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "058ecd4c",
   "metadata": {},
   "source": [
    "#### Helper functions for applying a single patch, and applying multiple patches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "96783171",
   "metadata": {},
   "outputs": [],
   "source": [
    "from eval_utils import predict_stuff\n",
    "import numpy as np\n",
    "import itertools\n",
    "\n",
    "def apply_patch_soft(exp_applies_probs, baseline_probs, label_clause):    \n",
    "    x = np.array([label_clause]).repeat(len(baseline_probs), 0)\n",
    "    #print(x.shape)\n",
    "    \n",
    "    applies_prob = exp_applies_probs[:, 1].reshape(-1, 1)\n",
    "    #print(applies_prob)\n",
    "    return applies_prob * x + (1 - applies_prob) * baseline_probs\n",
    "\n",
    "def get_scores_multiple_patches_hard(data, cond_list, examine=False):\n",
    "    no_exps = [('', ex) for ex in data[0]]\n",
    "    no_exp_probs = predict_stuff(no_exps, [0]*len(no_exps), model_obj, 'p1', verbose=False, mode='task_predictor')\n",
    "    cond_probs = []\n",
    "    interpret_probs = []\n",
    "    \n",
    "    all_patched_probs = []\n",
    "    for idx, (cond, label_clause) in enumerate(cond_list):\n",
    "        print(\"Applying patch {}\".format(cond))\n",
    "        contextualized = [(cond, ex) for ex in data[0]]\n",
    "        output_probs = predict_stuff(contextualized, itertools.repeat(0), model_obj, 'p1', verbose=False)\n",
    "        cond_probs.append(np.log(output_probs[:, 1])) # log(p(c | x))\n",
    "        patched_probs = apply_patch_soft(output_probs, no_exp_probs, label_clause) #Pr(y | x, lp)    \n",
    "        all_patched_probs.append(patched_probs[:, 1])\n",
    "        \n",
    "    # pick best patch and apply it! \n",
    "    all_patched_probs = np.stack(all_patched_probs, axis=1) # D x P\n",
    "    cond_probs = np.stack(cond_probs, axis=1) # D x P\n",
    "    best_patches = np.argmax(cond_probs, axis=1) # D x l\n",
    "    \n",
    "    ptrue = np.array([p[idx] for p, idx in zip(all_patched_probs, best_patches)])\n",
    "    pfalse = 1.0 - ptrue\n",
    "    return no_exp_probs, np.stack([pfalse, ptrue]).T\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b3a5543",
   "metadata": {},
   "source": [
    "#### Results for Yelp-Stars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "05c46074",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset yelp_polarity (/u/scr/smurty/yelp_polarity/plain_text/1.0.0/a770787b2526bdcbfc29ac2d9beb8e820fbc15a03afd3ebc4fb9d8529de57544)\n"
     ]
    }
   ],
   "source": [
    "from data_fns import get_yelp_stars\n",
    "tests_yelp = {'yelp_stars': get_yelp_stars()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b128c764",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1428fe88876245d19ce73f4021762811",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applying patch review gives 1 or 2 stars\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ce9a36aed1b419c9f2fb53c095cabbf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applying patch review gives zero stars\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9fadb5dee63e4675bfb8262f309f909c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "no_exp, ours = get_scores_multiple_patches_hard(tests_yelp['yelp_stars'], \n",
    "                                 [('review gives 1 or 2 stars', [1,0]), ('review gives zero stars', [1,0])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8bd83638",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9360025220680959\n",
      "0.9451450189155107\n"
     ]
    }
   ],
   "source": [
    "data = tests_yelp['yelp_stars']\n",
    "print(np.mean(no_exp.argmax(axis=1) == data[1]))\n",
    "print(np.mean(ours.argmax(axis=1) == data[1]))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc189cdd",
   "metadata": {},
   "source": [
    "#### Results for WCR data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "853f7a80",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59417f75043e42118c3559b3d4663e88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applying patch review says fit is boxy\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd2d6251fdea429d95b0ea0279d91a5d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applying patch review contains words or phrases like needs to be returned\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d628058ef7354370bcf43a1ad06ba1f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Applying patch review contains words or phrases like needs to be exchanged\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "099ba2eebf4447c1badfcf7f85af4fa4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pickle\n",
    "import pickle\n",
    "with open('wcr.pickle', 'rb') as reader:\n",
    "    dataset = pickle.load(reader)\n",
    "    \n",
    "explanations = [('review says fit is boxy',[1,0]),\n",
    "               (\"review contains words or phrases like needs to be returned\", [1, 0]),\n",
    "               (\"review contains words or phrases like needs to be exchanged\", [1,0])]\n",
    "    \n",
    "no_exp, ours = get_scores_multiple_patches_hard(dataset, explanations)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b01f78c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8890030832476875\n",
      "0.9009934909215485\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(no_exp.argmax(axis=1) == dataset[1]))\n",
    "print(np.mean(ours.argmax(axis=1) == dataset[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53cc9255",
   "metadata": {},
   "source": [
    "#### Tables 3 and Tables 4 (Controlling the model with patches on Yelp.)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d511b104",
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'compare_model_steering_labeled.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-6bb8911bdc66>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mdata_fns\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mget_yelp_data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m# set conflicting to True for Table-4 and False for Table-3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0md1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_yelp_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconflicting\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/juice2/scr2/smurty/LanguagePatching/data_fns.py\u001b[0m in \u001b[0;36mget_yelp_data\u001b[0;34m(conflicting)\u001b[0m\n\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_yelp_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconflicting\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m     \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'compare_model_steering_labeled.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m     \u001b[0mfix_quotes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m\"'\"\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m     \u001b[0mdf_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfix_quotes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcinput\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcinput\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Input'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/u/nlp/anaconda/main/anaconda3/envs/shikhar-basic/lib/python3.8/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001b[0m\n\u001b[1;32m    608\u001b[0m     \u001b[0mkwds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    609\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 610\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    611\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    612\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/u/nlp/anaconda/main/anaconda3/envs/shikhar-basic/lib/python3.8/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    460\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    461\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 462\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    463\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    464\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/u/nlp/anaconda/main/anaconda3/envs/shikhar-basic/lib/python3.8/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    817\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    818\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 819\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    820\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    821\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/u/nlp/anaconda/main/anaconda3/envs/shikhar-basic/lib/python3.8/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1048\u001b[0m             )\n\u001b[1;32m   1049\u001b[0m         \u001b[0;31m# error: Too many arguments for \"ParserBase\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1050\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mmapping\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[call-arg]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1051\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1052\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_failover_to_python\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/u/nlp/anaconda/main/anaconda3/envs/shikhar-basic/lib/python3.8/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   1865\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1866\u001b[0m         \u001b[0;31m# open handles\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1867\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open_handles\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1868\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandles\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1869\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m\"storage_options\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"encoding\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"memory_map\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"compression\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/u/nlp/anaconda/main/anaconda3/envs/shikhar-basic/lib/python3.8/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_open_handles\u001b[0;34m(self, src, kwds)\u001b[0m\n\u001b[1;32m   1360\u001b[0m         \u001b[0mLet\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mreaders\u001b[0m \u001b[0mopen\u001b[0m \u001b[0mIOHanldes\u001b[0m \u001b[0mafter\u001b[0m \u001b[0mthey\u001b[0m \u001b[0mare\u001b[0m \u001b[0mdone\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtheir\u001b[0m \u001b[0mpotential\u001b[0m \u001b[0mraises\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1361\u001b[0m         \"\"\"\n\u001b[0;32m-> 1362\u001b[0;31m         self.handles = get_handle(\n\u001b[0m\u001b[1;32m   1363\u001b[0m             \u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1364\u001b[0m             \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/u/nlp/anaconda/main/anaconda3/envs/shikhar-basic/lib/python3.8/site-packages/pandas/io/common.py\u001b[0m in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m    645\u001b[0m                 \u001b[0merrors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"replace\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    646\u001b[0m             \u001b[0;31m# Encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 647\u001b[0;31m             handle = open(\n\u001b[0m\u001b[1;32m    648\u001b[0m                 \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    649\u001b[0m                 \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'compare_model_steering_labeled.csv'"
     ]
    }
   ],
   "source": [
    "from data_fns import get_yelp_data\n",
    "# set conflicting to True for Table-4 and False for Table-3\n",
    "d1 = get_yelp_data(conflicting=False)\n",
    "print(len(d1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac8a014c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cond2label_dict(cond, orig_label):\n",
    "    is_food = 'food' in cond\n",
    "    is_good = 'good' in cond\n",
    "    \n",
    "    label_name2label = {'positive': 1, 'negative': 0, 'NAN': -1}\n",
    "    if is_food:\n",
    "        dict_to_use = [label for label in orig_label if label['category'] == 'food'][0]\n",
    "    else:\n",
    "        dict_to_use = [label for label in orig_label if label['category'] == 'service'][0]\n",
    "    # aspect sentiment. does patch apply\n",
    "    return label_name2label[dict_to_use['polarity']], int(label_name2label[dict_to_use['polarity']] == is_good)\n",
    "\n",
    "\n",
    "conds = [('food is good', [0,1]), ('service is good', [0,1]), ('food is bad',[1,0]), ('service is bad',[1,0])]\n",
    "label_sets = {cond: [cond2label_dict(cond, l) for l in d1[1]] for cond, _ in conds}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad65db33",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_steering_acc(data, labels, cond_labels, cond, cons, use_exps=True):\n",
    "    no_exps = [('', ex) for ex in data]\n",
    "    no_exp_probs = predict_stuff(no_exps, [0]*len(no_exps), model_obj, 'p1', verbose=False, mode='task_predictor')\n",
    "    no_exp_preds = no_exp_probs.argmax(axis=1)   \n",
    "    \n",
    "    if not use_exps:\n",
    "        acc_1 = np.sum((no_exp_preds == labels) & cond_labels)\n",
    "        return acc_1, np.sum(1-cond_labels), np.sum(cond_labels), np.sum(1-cond_labels)\n",
    "    else:\n",
    "        contextualized = [(cond, ex) for ex in data]\n",
    "        output_probs = predict_stuff(contextualized, cond_labels, model_obj, 'p1', verbose=False)\n",
    "        patched_probs = apply_patch_soft(output_probs, no_exp_probs, cons) #Pr(y | x, lp)\n",
    "        patched_preds = patched_probs.argmax(axis=1)\n",
    "        \n",
    "        # if patch applies, how often is model correct\n",
    "        acc_1 = np.sum((patched_preds == labels) & cond_labels)\n",
    "    \n",
    "        # if the patch doesn't apply, how often does the prediction say the same\n",
    "        acc_2 = np.sum((patched_preds == no_exp_preds) & (1-cond_labels))\n",
    "        return acc_1, acc_2, np.sum(cond_labels), np.sum(1-cond_labels)\n",
    "\n",
    "\n",
    "def get_scores(conds, use_exps=True):\n",
    "    t1 = 0.0\n",
    "    t2 = 0.0\n",
    "\n",
    "    total1 = 0.0\n",
    "    total2 = 0.0\n",
    "\n",
    "    for cond, cons in conds:\n",
    "        curr = label_sets[cond]\n",
    "        aspect_labels = np.array([a for a, _ in curr])\n",
    "        cond_applies = np.array([ca for _, ca in curr])\n",
    "\n",
    "        print(cond)\n",
    "        t1_c, t2_c, total1_c, total2_c = get_steering_acc(d1[0], aspect_labels, cond_applies, cond, cons, use_exps=use_exps)\n",
    "        t1 += t1_c\n",
    "        t2 += t2_c\n",
    "        total1 += total1_c\n",
    "        total2 += total2_c    \n",
    "    return t1 / total1, t2 / total2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa055d5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "s1, s2 = get_scores(conds)\n",
    "print(s1,s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d47a0f6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "s1, s2 = get_scores(conds, use_exps=False)\n",
    "print(s1, s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e8b9eb5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "shikhar-basic",
   "language": "python",
   "name": "shikhar-basic"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}