Explore_ISAN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "oebstyxGaBhh"
   },
   "outputs": [],
   "source": [
    "Load_saved = True\n",
    "\n",
    "save_dir = '/tmp/ISAN/'\n",
    "load_dir = '' #change this the the data dir "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "286cCOIGX0_D"
   },
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "No module named pandas",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-8753233eb72e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: No module named pandas"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib\n",
    "\n",
    "import math\n",
    "import os.path\n",
    "import time\n",
    "from sklearn.cluster import KMeans \n",
    "\n",
    "import scipy as sp #.spatial.distance.cosine(u, v)\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import copy\n",
    "import math\n",
    "import os.path\n",
    "import time\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import pickle\n",
    "import pandas\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "import seaborn\n",
    "\n",
    "\n",
    "seaborn.set_style(\"whitegrid\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "rp9sJLASvgtO"
   },
   "outputs": [],
   "source": [
    "#utility functions\n",
    "def normalize_logits(logits, axis=-1):\n",
    "  logits = logits - np.max(logits, axis=axis, keepdims=True)\n",
    "  logits -= np.log(np.exp(logits).sum(axis=axis, keepdims=True))\n",
    "  return logits\n",
    "\n",
    "def xentropy(p_lp, q_lp, axis=-1):\n",
    "  p_lp = normalize_logits(p_lp)\n",
    "  q_lp = normalize_logits(q_lp)\n",
    "  return -(np.exp(p_lp) * q_lp).sum(axis=axis)\n",
    "\n",
    "def entropy(p_lp, axis=-1):\n",
    "  p_lp = normalize_logits(p_lp)\n",
    "  return -(np.exp(p_lp) * p_lp).sum(axis=axis)\n",
    "\n",
    "def kl(p_lp, q_lp, axis=-1):\n",
    "  p_lp = normalize_logits(p_lp)\n",
    "  q_lp = normalize_logits(q_lp)\n",
    "  return (np.exp(p_lp) * (p_lp - q_lp)).sum(axis=axis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "3AncAftJ08IM"
   },
   "outputs": [],
   "source": [
    "#more utility\n",
    "import string\n",
    "chars = [\" \"]\n",
    "for c in string.ascii_lowercase:\n",
    "  chars.append(c)\n",
    "print chars\n",
    "\n",
    "pretty_chars = copy.copy(chars)\n",
    "pretty_chars[0] = \"_\"\n",
    "\n",
    "def get_chars(l):\n",
    "  c_list = [chars[i] for i in l]\n",
    "  return c_list\n",
    "\n",
    "\n",
    "mapping = {}\n",
    "for i in range(27):\n",
    "  mapping[chars[i]] = i\n",
    "\n",
    "def standard_figure(width = 9, height = 3.2):\n",
    "  fig = plt.figure(figsize=(width, height), dpi=300) #im\n",
    "  return fig\n",
    "\n",
    "                            \n",
    "def soft_max(x, t = 1):\n",
    "  return np.exp(x*t) / np.sum( np.exp(x*t))\n",
    "\n",
    "def make_step(h, input_val, target = None, multiplier = 1):\n",
    "  h = np.dot( w_x[input_val], np.reshape( h, [-1,1] )) + np.reshape(b_ixn[input_val], [-1,1] )*multiplier\n",
    "  output = np.dot( np.reshape(h, [1,-1]), w_out) + out_bias\n",
    "  if not target is None:\n",
    "    loss = sess.run( [train_loss_np], feed_dict={output_placeholder: output, target_placeholder:target})[0]\n",
    "    return h, output, loss\n",
    "  else:\n",
    "    return h, output\n",
    "  \n",
    "def make_step_back(h, input_val):\n",
    "  h = np.transpose( np.dot( np.linalg.inv( w_x[input_val]), np.reshape( h, [-1,1] )  - np.reshape(b_ixn[input_val], [-1,1] )) )\n",
    "  prob = soft_max(h.dot(w_out))\n",
    "  return h, prob[0][input_val]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "tYVbaLqgqgiW"
   },
   "outputs": [],
   "source": [
    "true_num_nodes = 27 * 216**2 + 2 * 27 * 216 + 216 + 27\n",
    "print true_num_nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "OwNaEk253Sjg"
   },
   "outputs": [],
   "source": [
    "#Parameters - set as required\n",
    "input_size = output_size = 27\n",
    "num = 1280000\n",
    "num_nodes = 216\n",
    "snapshop = \"\" #\"100001\"\n",
    "hpvalues = {}\n",
    "batch_size = 1\n",
    "\n",
    "n_random = 10000  #number of random states to have\n",
    "\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "input_placeholder = tf.placeholder(tf.float32, shape=[None, input_size])\n",
    "target_placeholder = tf.placeholder(tf.float32, shape=[None, input_size])\n",
    "state_placeholder = tf.placeholder(tf.float32, shape=[None, num_nodes])\n",
    "output_placeholder = tf.placeholder(tf.float32, shape=[None, 27])\n",
    "\n",
    "\n",
    "train_loss_np  = tf.reduce_mean(\n",
    "      tf.nn.softmax_cross_entropy_with_logits(\n",
    "        output_placeholder, target_placeholder) / tf.log(2.0) )\n",
    "\n",
    "\n",
    "temperature = tf.placeholder(tf.float32, shape=[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "deB9qobqiI6w"
   },
   "outputs": [],
   "source": [
    "with open(os.path.join(load_dir, \"x_y.pkl\")) as pkl_file:\n",
    "  x, y = pickle.load(pkl_file)\n",
    "\n",
    "with open(os.path.join(load_dir, \"wx_bix_wo_hz_ob.pkl\")) as pkl_file:\n",
    "  w_x, b_ixn, w_out, h_zero, out_bias = pickle.load(pkl_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "a0ykYoUJ51JR"
   },
   "outputs": [],
   "source": [
    "# CREATE PCA fit for figure 1\n",
    "\n",
    "losses = []\n",
    "count = 0\n",
    "h  = h_zero\n",
    "counter = 0\n",
    "h_space = h\n",
    "h_other = h\n",
    "random_states = []\n",
    "losses = []\n",
    "for i in range(10000):\n",
    "  h_in = h\n",
    "  input_val = np.argmax(x[i])\n",
    "  h, outpt_np, loss =  make_step(h, input_val, y[i])\n",
    "  \n",
    "  if i % 100 == 0:\n",
    "    h  = h_zero\n",
    "  if i %100 > 10:\n",
    "    random_states.append(h[:,0])\n",
    "    losses.append(loss)\n",
    "print(np.mean(losses))\n",
    "\n",
    "pca_dim = num_nodes\n",
    "pca = PCA(n_components=pca_dim)\n",
    "pca.fit(random_states[:])\n",
    "\n",
    "print len(pca.explained_variance_ratio_)\n",
    "pca_basis = pca.components_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "lC5K3MZF0CMR"
   },
   "outputs": [],
   "source": [
    "#pca plots for different architectures\n",
    "with open(os.path.join(load_dir, \"gru_308_evratio.pkl\")) as pkl_file:\n",
    "  gru308_ev = pickle.load(pkl_file)\n",
    "  print gru308_ev[:5], len(gru308_ev)\n",
    "with open(os.path.join(load_dir, \"gru_145_evratio.pkl\")) as pkl_file:\n",
    "  gru145_ev = pickle.load(pkl_file)\n",
    "  print gru145_ev[:5], len(gru145_ev)\n",
    "with open(os.path.join(load_dir, \"rnn_256_evratio.pkl\")) as pkl_file:\n",
    "  rnn256_ev = pickle.load(pkl_file)\n",
    "  print rnn256_ev[:5], len(rnn256_ev)\n",
    "with open(os.path.join(load_dir, \"rnn_116_evratio.pkl\")) as pkl_file:\n",
    "  rnn116_ev = pickle.load(pkl_file)\n",
    "  print rnn116_ev[:5], len(rnn116_ev)\n",
    " \n",
    "standard_figure(width = 3.5, height=2.7)\n",
    "#fig = plt.figure(figsize=(11, height), dpi=300) #im\n",
    "plt.semilogy(gru308_ev, label = \"GRU-308\")\n",
    "plt.semilogy(gru145_ev, label = \"GRU-145\")\n",
    "plt.semilogy(rnn256_ev, label = \"RNN-256\")\n",
    "plt.semilogy(rnn116_ev, label = \"RNN-116\")\n",
    "plt.semilogy(pca.explained_variance_ratio_, label = \"ISAN-216\")\n",
    "plt.ylim(.5e-3, .2)\n",
    "plt.xlim(0, 210)\n",
    "plt.ylabel(\"Explained variance\")\n",
    "plt.xlabel(\"PCA dim\")\n",
    "plt.grid(False)\n",
    "plt.legend()\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig(save_dir + \"pca_ev_plot_narrow.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "KnksXtNmQ7Kq"
   },
   "outputs": [],
   "source": [
    "\n",
    "#Fig1 / Master_plot\n",
    "string = \" annual revenue \"\n",
    "#string = \" a lit of \"\n",
    "zoom_idx = 11\n",
    "\n",
    "fig = standard_figure()\n",
    "\n",
    "history = []\n",
    "history_cont = []\n",
    "max_history = len(string)\n",
    "out_history = np.ones((max_history, 27))\n",
    "space_norms = np.zeros(max_history)\n",
    "first_letter = np.zeros(max_history)\n",
    "\n",
    "canvas = np.zeros((max_history*28, 2*max_history + 4))*np.nan\n",
    "probabilities = np.zeros((max_history*28,1))\n",
    "  \n",
    "df_limit_history = pd.DataFrame()\n",
    "h  = np.zeros((num_nodes,1))\n",
    "#history.append(h)\n",
    "df_limit_history = pd.DataFrame()\n",
    "\n",
    "count = 0\n",
    "for i in range(max_history):\n",
    "  if i > 0:\n",
    "    h_in = h\n",
    "  else:\n",
    "    h_in = np.zeros((num_nodes,1))\n",
    "  input_val = mapping[string[i]]\n",
    "  sum_of_hm1_cont = np.zeros((num_nodes,1))\n",
    "\n",
    "  for k in range(len(history)):\n",
    "    new_h = np.dot( w_x[input_val], np.reshape( history[k], [-1,1] ))\n",
    "    history[k] = new_h\n",
    "    sum_of_hm1_cont = sum_of_hm1_cont + new_h\n",
    "\n",
    "  new_offset = np.reshape(b_ixn[input_val], [-1,1])\n",
    "  history.append(new_offset)\n",
    "  h = sum_of_hm1_cont +  new_offset\n",
    "  \n",
    "  history_cont = []\n",
    "  for k in range(len(history)):\n",
    "    new_cont = np.dot( np.reshape(history[k], [1,-1]), w_out)\n",
    "    canvas[i*28 : (i) * 28 + 27,(k)*2:(k+1)*2] = np.reshape( new_cont, [-1, 1])\n",
    "    history_cont.append( new_cont )\n",
    "  space_norms[i] = np.linalg.norm( history_cont[0]) \n",
    "  if i > 0:\n",
    "    first_letter[i] = np.linalg.norm( history_cont[1]) \n",
    "  outpt_np = np.dot( np.reshape(h, [1,-1]), w_out) + out_bias\n",
    "  probabilities[i*28 : (i) * 28+27] = np.reshape( soft_max(outpt_np) ,[-1,1])\n",
    "  \n",
    "  canvas[i*28 : (i) * 28 + 27,-2:]  =  np.reshape(outpt_np, [-1,1])\n",
    "  canvas[i*28 : (i) * 28 + 27,-4:-2]  = np.reshape(out_bias,[-1,1])\n",
    "  \n",
    "  if i+1 < len(string):\n",
    "    true_char = string[i+1]\n",
    "  else:\n",
    "    true_char = \"\"\n",
    "  loss = sess.run( [train_loss_np], feed_dict={output_placeholder: outpt_np, target_placeholder:y[i]})\n",
    "\n",
    "  d = {\"index\" : i, \"state\": h, \"input\":input_val, \"state_in\":h_in, \"delta\":h-h_in, \"loss\": loss, \"history\":history[:], \"output\": outpt_np, \"props\": soft_max(outpt_np), \"history_cont\": history_cont[:], \n",
    "       \"input_char\":string[i], \"predict_char\":chars[np.argmax(outpt_np)], \"true_char\": true_char }\n",
    "  df_limit_history = df_limit_history.append(d, ignore_index=True)\n",
    "\n",
    "\n",
    "\n",
    "#fig = standard_figure()\n",
    "fig = plt.figure(figsize=(15,5))\n",
    "\n",
    "#(ax1, ax2) = plt.subplots(2, sharex=True)  \n",
    "plt.subplot(2,1,1)    \n",
    "ax  = plt.gca()\n",
    "ax.patch.set_facecolor('white')\n",
    "plt.imshow(np.transpose(canvas), 'jet', interpolation = 'nearest', vmin = -5, vmax = 10)\n",
    "#plt.ylabel(\"$\\kappa^{t}_{s}$\",rotation = 'horizontal',fontsize =17 )\n",
    "#plt.text(-15,+10, \"$\\kappa^{t}_{s}$\",fontsize = 17 )\n",
    "plt.text(-15,+15, \"$s$\",fontsize = 17 )\n",
    "currentAxis = plt.gca()\n",
    "currentAxis.add_patch(matplotlib.patches.Rectangle((28*zoom_idx-1,-1), 28, len(canvas[0])+1, edgecolor=\"red\", fill = False, linewidth=3))\n",
    "\n",
    "ax.annotate(\"\",\n",
    "              xy=(0.05, 0.5), xycoords='figure fraction',\n",
    "              xytext=(0.07, 0.5), textcoords='figure fraction',\n",
    "              arrowprops=dict(arrowstyle=\"->\"),\n",
    "            )\n",
    "ax.annotate(\"\",\n",
    "              xy=(0.035, .7), xycoords='figure fraction',\n",
    "              xytext=(0.035, .9), textcoords='figure fraction',\n",
    "              arrowprops=dict(arrowstyle=\"->\"),\n",
    "            )\n",
    "\n",
    "_=plt.xticks( np.arange(1,max_history*28,28)  -2 )\n",
    "_=plt.yticks( np.arange(-1,-1,max_history) )\n",
    "plt.axis('tight')\n",
    "\n",
    "for i in range(max_history):\n",
    "  val =  string[i]\n",
    "  if val ==\" \":\n",
    "    val = \"_\"\n",
    "  plt.text(i*28 +14, -2.0 , val)\n",
    "plt.text(-20, -2.0 , 'input:')\n",
    "\n",
    "#fig.colorbar(ax)\n",
    "\n",
    "plt.subplot(2,1,2, sharex=ax )  \n",
    "plt.ylabel(\"Predicted next letter\")\n",
    "_ = plt.stem( probabilities, markerfmt='') \n",
    "# _ = plt.stem( probabilities, '.' ) \n",
    "_ = plt.xticks( np.arange(-1,max_history*28,28),[])\n",
    "_ = plt.yticks( np.arange(0,1.1,0.5) )\n",
    "\n",
    "\n",
    "for i in range(max_history):\n",
    "  val =   np.argmax(probabilities[i*28 : (i) *28 + 27])\n",
    "  plt.text(i*28 + val + 1, probabilities[i*28 + val] -0.07 , pretty_chars[val])\n",
    "  #for k in range(5):\n",
    "  #  plt.text(i*28 +k*5, -0.1 , pretty_chars[5*k])\n",
    "  #plt.text(i*28 +25, -0.1 , 'z')\n",
    "  plt.text(i*28 +14, 1.22, str(i+1) )\n",
    "plt.text(-20, 1.22, 'step:')\n",
    "plt.xlabel(\"$t$\", fontsize = 17)\n",
    "plt.tight_layout()\n",
    "plt.ylim(0, 1.2)\n",
    "#plt.tight_layout(pad=0.4, w_pad=0.5, h_pad=1.0)\n",
    "ax = plt.gca()\n",
    "ax.yaxis.grid(False)\n",
    "\n",
    "plt.savefig(save_dir + \"master_plot.pdf\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "sEvpeJOKeI1p"
   },
   "outputs": [],
   "source": [
    "#fig = standard_figure()\n",
    "fig = plt.figure(figsize=(15,5))\n",
    "canvas1 = canvas[:, ::2]\n",
    "canvas1 = canvas1.reshape((-1, 28, canvas1.shape[-1]))\n",
    "final_lp = canvas1[:, :-1, -1]\n",
    "canvas1 = canvas1[:,:-1,:-1]\n",
    "unigram_lp = canvas1[0,:,-1]\n",
    "\n",
    "canvas2 = np.zeros((canvas1.shape[0], canvas1.shape[-1]-1))\n",
    "\n",
    "letters = np.zeros(canvas2.shape)\n",
    "\n",
    "  \n",
    "aggregate_low = lambda i,j: j #from step j\n",
    "aggregate_high = lambda i,j: j+1 #till step j\n",
    "#aggregate_high = lambda i,j: i+1 #till step i\n",
    "\n",
    "measure = 'norm'\n",
    "measure = 'xentropy'\n",
    "measure = 'rev_xentropy'\n",
    "measure = 'rev_kl'\n",
    "measure = 'kl'\n",
    "\n",
    "measure = 'entropy'\n",
    "\n",
    "\n",
    "for i in range(canvas1.shape[0]):\n",
    "  cell = canvas1[i, :, :]\n",
    "  kl_divs = np.zeros((cell.shape[-1]-1, ))\n",
    "  first_err = -1\n",
    "  for j in xrange(kl_divs.shape[0]):\n",
    "    cum_log_probs = unigram_lp + cell[:,aggregate_low(i,j):aggregate_high(i,j)].sum(1)\n",
    "    letters[i, j] = np.argmax(cum_log_probs)\n",
    "    if measure == \"kl\":\n",
    "      kl_divs[j] = kl(final_lp[i, :], cum_log_probs)\n",
    "    elif measure == \"rev_kl\":\n",
    "      kl_divs[j] = kl(cum_log_probs, final_lp[i, :])\n",
    "    elif measure == \"norm\":\n",
    "      kl_divs[j] = np.linalg.norm(cum_log_probs)\n",
    "    elif measure == \"entropy\":\n",
    "      kl_divs[j] = entropy(cum_log_probs)\n",
    "    elif measure == \"xentropy\":\n",
    "      kl_divs[j] = xentropy(final_lp[i, :], cum_log_probs)\n",
    "    elif measure == \"rev_xentropy\":\n",
    "      kl_divs[j] = xentropy(cum_log_probs, final_lp[i, :])\n",
    "    else:\n",
    "      raise Exception(\"Unknown measure\")\n",
    "  kl_divs[i+1:] = np.nan\n",
    "  canvas2[i, :] = kl_divs\n",
    "\n",
    "plt.imshow(canvas2.T, 'Blues_r', interpolation='nearest', vmax=4)\n",
    "plt.grid(False)\n",
    "plt.axis('tight')\n",
    "plt.xticks(*zip(*enumerate(string[1:])))\n",
    "plt.yticks(*zip(*list(enumerate(string))))\n",
    "\n",
    "if len(string)< 20:\n",
    "  for i,j in zip(*np.nonzero(np.isfinite(canvas2))):\n",
    "    if i==len(string)-1:\n",
    "      continue\n",
    "    if int(letters[i,j]) == mapping[string[i+1]]:\n",
    "      color='k'\n",
    "    else:\n",
    "      color='r'\n",
    "    #plt.text(i,j+0.3,pretty_chars[int(letters[i,j])], color=color)\n",
    "plt.colorbar()\n",
    "\n",
    "#plt.savefig(save_dir + \"jan_entropy_plot.pdf\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "toN6fp0o5jyR"
   },
   "outputs": [],
   "source": [
    "#fig = plt.figure(figsize=(7, 7))\n",
    "\n",
    "index = zoom_idx\n",
    "hist = df_limit_history[\"history_cont\"][index]\n",
    "true_char = df_limit_history[\"true_char\"][index]\n",
    "#pred_char = df_limit_history[\"predict_char\"][index]\n",
    "pred_char = 'r'\n",
    "pred_char_idx = mapping[pred_char]\n",
    "true_char_idx = mapping[true_char]\n",
    "\n",
    "standard_figure()\n",
    "#fig = plt.figure(figsize=(11, 4), dpi=300)\n",
    "\n",
    "plt.subplot(1,3,1)\n",
    "\n",
    "p = plt.imshow(np.transpose(canvas[zoom_idx * 28: zoom_idx * 28 + 27, :]), 'jet', interpolation= 'nearest', vmin = -5, vmax = 10)\n",
    "\n",
    "currentAxis = plt.gca()\n",
    "currentAxis.add_patch(matplotlib.patches.Rectangle((pred_char_idx -0.5,-1), 1, len(string)*2+14, edgecolor=\"red\", fill = False, linewidth=1))\n",
    "currentAxis.add_patch(matplotlib.patches.Rectangle((true_char_idx -0.5,-1), 1, len(string)*2+14, edgecolor=\"orange\", fill = False, linewidth=1))\n",
    "\n",
    "plt.xticks(np.arange(0, 27), chars, fontsize = 8)\n",
    "label_chars = df_limit_history[\"input_char\"][:zoom_idx+1]\n",
    "for i in xrange(len(label_chars)):\n",
    "  if label_chars[i] == \" \" or label_chars[i] == \"''\":\n",
    "    label_chars[i] = \"_\"\n",
    "plt.title(\"a)\")\n",
    "#plt.yticks(2*np.arange(zoom_idx+1) + 0.5, label_chars)\n",
    "plt.yticks( np.arange(-1,-1,max_history) )\n",
    "plt.xlabel(\"Output character\")\n",
    "plt.text(-7, 10, \"$\\kappa_s^{12}$\", fontsize = 17 )\n",
    "plt.grid(False)\n",
    "plt.colorbar(p)\n",
    "\n",
    "\n",
    "\n",
    "print true_char_idx, true_char, pred_char\n",
    "\n",
    "values = [hist[i][0][true_char_idx] for i in xrange(len(hist))]\n",
    "values_pred = [hist[i][0][pred_char_idx] for i in xrange(len(hist))]\n",
    "\n",
    "max_val = np.max([np.max(values), np.max(values_pred)]) + 0.5\n",
    "min_val = np.min([np.min(values), np.min(values_pred)]) - 0.5\n",
    "\n",
    "for i in xrange(len(values)):\n",
    "  print df_limit_history[\"input_char\"][i], values[i]\n",
    "\n",
    "ind = np.arange(len(hist))    # the x locations for the groups\n",
    "width = 1.0       # the width of the bars: can also be len(x) sequence\n",
    "\n",
    "summed = np.zeros(27)\n",
    "#positive_summed = np.zeros(27)\n",
    "\n",
    "plt.subplot(1,3,2)\n",
    "\n",
    "p = plt.bar(ind, values, width/2.5, color = 'orange', label = 'n logit')\n",
    "p = plt.bar(ind + width/2.5, values_pred, width/2.5, color = 'red', label = 'r logit')\n",
    "\n",
    "plt.title(\"b)\")\n",
    "#plt.title(\"Contribution to '{}' logit\".format(true_char))\n",
    "_ = plt.xticks( np.arange(0,len(hist),1) + 0.5, label_chars)\n",
    "plt.ylim(min_val, max_val)\n",
    "plt.grid(False)\n",
    "\n",
    "plt.legend(loc=0)\n",
    "height = max(values)\n",
    "\n",
    "input_chars = [df_limit_history[\"input_char\"][i] for i in xrange(len(hist))]\n",
    "\n",
    "values = [hist[i][0][true_char_idx] for i in xrange(len(hist))]\n",
    "values_pred = [hist[i][0][pred_char_idx] for i in xrange(len(hist))]\n",
    "\n",
    "\n",
    "ind = np.arange(len(hist))    # the x locations for the groups\n",
    "width = 1.0       # the width of the bars: can also be len(x) sequence\n",
    "\n",
    "\n",
    "sum_idx1 = 7\n",
    "true_char_sum1 = np.sum(values[:sum_idx1])\n",
    "true_char_sum2 = np.sum(values[sum_idx1:])\n",
    "pred_char_sum1 = np.sum(values_pred[:sum_idx1])\n",
    "pred_char_sum2 = np.sum(values_pred[sum_idx1:])\n",
    "\n",
    "p_values1 = [true_char_sum1, true_char_sum2, true_char_sum1+true_char_sum2]\n",
    "p_values2 = [pred_char_sum1, pred_char_sum2, pred_char_sum1 + pred_char_sum2]\n",
    "\n",
    "plot_ind1 = [1, 4, 7]\n",
    "plot_ind2 = [2, 5, 8]\n",
    "\n",
    "ax = plt.gca()\n",
    "\n",
    "ax.annotate(\"\",\n",
    "              xy=(0.035, .4), xycoords='figure fraction',\n",
    "              xytext=(0.035, .6), textcoords='figure fraction',\n",
    "              arrowprops=dict(arrowstyle=\"->\"),\n",
    "            )\n",
    "\n",
    "ax.annotate(\"\",\n",
    "              xy=(0.58, .08), xycoords='figure fraction',\n",
    "              xytext=(0.51, .08), textcoords='figure fraction',\n",
    "              arrowprops=dict(arrowstyle=\"->\"),\n",
    "            )\n",
    "plt.xlabel(\"s       \")\n",
    "\n",
    "plt.subplot(1,3,3)\n",
    "\n",
    "p = plt.bar(plot_ind1, p_values1, width, color = 'orange', label = \"n logit\")\n",
    "p = plt.bar(plot_ind2, p_values2, width, color = 'red', label =  'r logit')\n",
    "plt.title(\"c)\")\n",
    "plt.legend(loc =0)\n",
    "#plt.title(\"Contribution to '{}' logit\".format(pred_char))\n",
    "plt.grid(False)\n",
    "_ = plt.xticks( [1.5,3, 4.2,5.5, 7.5], [\"_annual\", \"+\", \"_reve\", \"=\", \"_annual_reve\"])\n",
    "plt.text(3.3, 1.5, \"+\")\n",
    "plt.text(6.2, 1.5, \"=\")\n",
    "# plt.text(1.0, -1, \"$\\kappa_{\\_annual}^{11}$\")\n",
    "# plt.text(4.5, -1, \"$\\kappa_{\\_reve}^{11}$\")\n",
    "# plt.text(4.5, -1, \"$\\kappa_{\\_reve}^{11}$\")\n",
    "#plt.ylim(min_val, max_val)\n",
    "height = max(values)\n",
    "\n",
    "\n",
    "\n",
    "# plt.tight_layout()  \n",
    "plt.tight_layout(w_pad=-.2)  \n",
    "plt.savefig(save_dir + \"zoom_in_plot.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "20kwJgfB5llg"
   },
   "outputs": [],
   "source": [
    "#fig2 NORM OF ELEMENTS IN THE BUFFER\n",
    "errors = np.zeros(50)\n",
    "#h  = np.zeros((1, num_nodes))\n",
    "h  = h_zero\n",
    "losses = []\n",
    "counter = 0\n",
    "max_history = 200\n",
    "history = [0 for i in xrange(max_history)]\n",
    "input_history = [0 for i in xrange(max_history)]\n",
    "df_limit_history = pd.DataFrame()\n",
    "\n",
    "count = 0\n",
    "for i in range(10000,15000):\n",
    "  if i % 1000 == 0:\n",
    "    print i, len(df_limit_history)\n",
    "  h_in = h\n",
    "  input_val = np.argmax(x[i])\n",
    "  for k in range(min(max_history, count)):\n",
    "    new_h = np.dot( w_x[input_val], np.reshape( history[k], [-1,1] ))\n",
    "    out_val = out_val + new_h\n",
    "    history[k] = new_h\n",
    "    \n",
    "  out_val = np.zeros((num_nodes,1))\n",
    "  for k in range(min(max_history, count)):\n",
    "    if not k > max_history:\n",
    "      out_val = out_val +  history[k]\n",
    "  h = out_val +  np.reshape(b_ixn[input_val], [-1,1] ) \n",
    "  if len( history) == max_history: \n",
    "    history[-1] = history[-1] + history[-2]\n",
    "    history.pop(-2)\n",
    "    input_history.pop(-2)\n",
    "  history.insert(0,np.reshape(b_ixn[input_val], [-1,1] ) )\n",
    "  input_history.insert(0,input_val)\n",
    "  outpt_np = np.dot( np.reshape(h, [1,-1]), w_out) + out_bias\n",
    "  loss = sess.run( [train_loss_np], feed_dict={output_placeholder: outpt_np, target_placeholder:y[i]})\n",
    "  count += 1\n",
    "  if i  > 10:\n",
    "    losses.append(loss)\n",
    "    d = {\"index\" : i, \"state\": h, \"input\":np.argmax(x[i]), \"state_in\":h_in, \"delta\":h-h_in, \"dist\": i % 50, \"loss\": loss, \"history\":history[:], \"input_history\": input_history[:] }\n",
    "    df_limit_history = df_limit_history.append(d, ignore_index=True)\n",
    "print(\"produced {} random states\".format(n_random))\n",
    "print(np.mean(losses))\n",
    "\n",
    "\n",
    "#dropping the most recent offset from the current prediction reduces accuracy to 1.9, droppping the 1st contribution to 1.718, 2nd 1.66771"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "BCahFmq85VSf"
   },
   "outputs": [],
   "source": [
    "#Calculate norm of various components in the buffer\n",
    "vals = np.zeros((len(df_limit_history),max_history, 27))\n",
    "logit_val = np.zeros((len(df_limit_history),max_history, 27))\n",
    "length = np.zeros((len(df_limit_history),max_history, 27))\n",
    "logit_length = np.zeros((len(df_limit_history),max_history, 27))\n",
    "for i in range(len(df_limit_history)):\n",
    "  i_vec = np.zeros((num_nodes,1))\n",
    "  for j in range( min(max_history, i)-1, -1, -1):\n",
    "    save_dex = df_limit_history[\"input_history\"][i][j]\n",
    "    vals[i,j, save_dex] = np.linalg.norm(df_limit_history[\"history\"][i][j])\n",
    "    i_vec = i_vec + df_limit_history[\"history\"][i][j]\n",
    "    length[i,j, save_dex] = np.linalg.norm(i_vec)\n",
    "    logit_length[i,j, save_dex] = np.linalg.norm(np.dot( np.reshape(i_vec, [1,-1]), w_out) )\n",
    "    logit_val[i,j, save_dex] = np.linalg.norm( np.dot(np.reshape(df_limit_history[\"history\"][i][j], [1,-1]), w_out))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "OwuARabG7Oa0"
   },
   "outputs": [],
   "source": [
    "#FIGURE DECAY AND IMPORTANCE STARTS HERE\n",
    "#spacebar #Fig5 #importance of space bar\n",
    "\n",
    "#get weights\n",
    "fig = plt.figure(figsize=(20, 15))\n",
    "df_limit_history_space = pd.DataFrame()\n",
    "losses = []\n",
    "count = 0\n",
    "h  = h_zero\n",
    "counter = 0\n",
    "h_space = h\n",
    "h_other = h\n",
    "for i in range(10000):\n",
    "  h_in = h\n",
    "  input_val = np.argmax(x[i])\n",
    "  if input_val == 0:\n",
    "    multiplier_space = 1\n",
    "    multiplier_other = 0\n",
    "    counter = 0\n",
    "  else:    \n",
    "    multiplier_space = 0\n",
    "    multiplier_other = 1    \n",
    "    counter = counter + 1\n",
    "  h_space, outpt_np_space, loss_space = make_step(h_space, input_val, y[i], multiplier_space)\n",
    "  h_other, outpt_np_other, loss_other = make_step(h_other, input_val, y[i], multiplier_other)\n",
    "  h, outpt_np, loss =  make_step(h, input_val, y[i])\n",
    "\n",
    "  d = {\"index\" : i, \"input\":input_val, \"state_in\":h_in, \"counter\":counter, \"loss\": loss, \"loss_space\": loss_space, \"loss_other\": loss_other, \"predict_char\":chars[np.argmax(outpt_np)], \"true_char\": true_char }\n",
    "  df_limit_history_space = df_limit_history_space.append(d, ignore_index=True)\n",
    "\n",
    "  if i > 10:\n",
    "    losses.append(loss)\n",
    "print(np.mean(losses))\n",
    "\n",
    "space_only = df_limit_history_space.groupby(['counter'])[\"loss_space\"].median().values\n",
    "without_space = df_limit_history_space.groupby(['counter'])[\"loss_other\"].median().values\n",
    "all_chars = df_limit_history_space.groupby(['counter'])[\"loss\"].median().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "ngZiEi3W2dyP"
   },
   "outputs": [],
   "source": [
    "#norm_decay\n",
    "fig =standard_figure()\n",
    "#plt.plot(length.mean(0))\n",
    "x_lim = 20\n",
    "\n",
    "x_axis = np.arange(x_lim+1)\n",
    "\n",
    "plt.subplot(1,3,1)\n",
    "#plt.title(\"Norm of logit contribution\")\n",
    "plt.title(\"a)\")\n",
    "for i in range(27):\n",
    "  normalising = np.sum(( logit_val[:,:x_lim+1,i]  != 0), 0 )\n",
    "  y_s =  (logit_val[:,:,i].sum(0)[:x_lim+1]) /normalising\n",
    "  if i < 5:\n",
    "    plt.semilogy(x_axis, y_s, label = pretty_chars[i])\n",
    "  elif i ==5:\n",
    "    plt.semilogy(x_axis, y_s, label = \"...\")\n",
    "  else:\n",
    "    plt.semilogy(x_axis, y_s)\n",
    "    # plt.semilogy(x_axis, (logit_val[:,:,0].mean(0)[:x_lim]), 'g', label = 'space')\n",
    "# plt.semilogy(x_axis, (logit_val[:,:,1].mean(0)[:x_lim]),'r', label = 'not space')\n",
    "plt.xlabel('Number of steps')\n",
    "plt.ylabel('Norm of logit vector')\n",
    "plt.legend()\n",
    "\n",
    "plt.subplot(1,3,2)\n",
    "#plt.title(\"Norm of logit contribution\")\n",
    "plt.title(\"b)\")\n",
    "temp = np.sum(logit_val,2)\n",
    "plt.semilogy(temp.mean(0)[:-100], 'g', label = 'all chars')\n",
    "#plt.loglog((logit_val[:,:,1].mean(0)[:-1]),'r', label = 'not space')\n",
    "plt.xlabel('Number of steps')\n",
    "#plt.ylabel('norm of logit vector')\n",
    "plt.legend()\n",
    "\n",
    "plt.subplot(1,3,3)\n",
    "\n",
    "plt.plot(without_space[:14], label = 'no space')\n",
    "plt.plot(space_only[:14], 'g', label = 'space only')\n",
    "plt.plot(all_chars[:14], 'r', label = 'all')\n",
    "plt.xlabel('Position in word')\n",
    "plt.ylabel('Median perplexity')\n",
    "#plt.title('Perplexity by position in word')\n",
    "plt.title(\"c)\")\n",
    "\n",
    "plt.legend()\n",
    "\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig(save_dir + \"norm_decay.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "OgYygWOiZhe_"
   },
   "outputs": [],
   "source": [
    "#Fig4 / Master_plot by words\n",
    "#string = \" the united states of america caused a lot of conflict \"\n",
    "string = \" the annual revenue was higher than expected \"\n",
    "n_words = len(string.split(\" \"))-1\n",
    "fig = standard_figure()\n",
    "\n",
    "history = []\n",
    "history_cont = []\n",
    "max_history = len(string)\n",
    "out_history = np.ones((max_history, 27))\n",
    "space_norms = np.zeros(max_history)\n",
    "first_letter = np.zeros(max_history)\n",
    "\n",
    "canvas = np.zeros((max_history*28, 2*n_words + 4))\n",
    "probabilities = np.zeros((max_history*28,1))\n",
    "  \n",
    "df_limit_history = pd.DataFrame()\n",
    "h  = np.zeros((num_nodes,1))\n",
    "#history.append(h)\n",
    "df_limit_history = pd.DataFrame()\n",
    "\n",
    "range_temp =  len( canvas[1,:])/2\n",
    "superprobcanvas = np.zeros((max_history,range_temp))\n",
    "\n",
    "\n",
    "count = 0\n",
    "for i in range(max_history):\n",
    "  if i > 0:\n",
    "    h_in = h\n",
    "  else:\n",
    "    h_in = np.zeros((num_nodes,1))\n",
    "  input_val = mapping[string[i]]\n",
    "  sum_of_hm1_cont = np.zeros((num_nodes,1))\n",
    "  sum_of_hm1_cont_super =  np.reshape(b_ixn[input_val], [-1,1])\n",
    "  for k in range(len(history)-1, -1, -1):\n",
    "    new_h = np.dot( w_x[input_val], np.reshape( history[k], [-1,1] ))\n",
    "    history[k] = new_h\n",
    "    sum_of_hm1_cont = sum_of_hm1_cont + new_h\n",
    "    \n",
    "    sum_of_hm1_cont_super = sum_of_hm1_cont_super + np.reshape(new_h, [-1,1])\n",
    "    if i < (len(string) - 1):\n",
    "      temp = np.dot( np.reshape(sum_of_hm1_cont_super, [1,-1]), w_out) + out_bias\n",
    "      superprobcanvas[i, k] = np.reshape( soft_max(temp ) ,[-1])[mapping[string[i+1]]]\n",
    "      #print sum_of_hm1_cont_super.sum()\n",
    "      #print     superprobcanvas[i, k]\n",
    "    \n",
    "\n",
    "  new_offset = np.reshape(b_ixn[input_val], [-1,1])\n",
    "  if input_val == 0:\n",
    "    history.append(new_offset)\n",
    "  else:\n",
    "    history[-1] = history[-1]+new_offset\n",
    "  h = sum_of_hm1_cont +  new_offset\n",
    "  \n",
    "  history_cont = []\n",
    "  for k in range(len(history)):\n",
    "    new_cont = np.dot( np.reshape(history[k], [1,-1]), w_out)\n",
    "    canvas[i*28 : (i) * 28 + 27,(k)*2:(k+1)*2] = np.reshape( new_cont, [-1, 1])\n",
    "    history_cont.append( new_cont )\n",
    "  space_norms[i] = np.linalg.norm( history_cont[0]) \n",
    "  outpt_np = np.dot( np.reshape(h, [1,-1]), w_out) + out_bias\n",
    "  probabilities[i*28 : (i) * 28+27] = np.reshape( soft_max(outpt_np) ,[-1,1])\n",
    "  \n",
    "  canvas[i*28 : (i) * 28 + 27,-2:]  =  np.reshape(outpt_np, [-1,1])\n",
    "  canvas[i*28 : (i) * 28 + 27,-4:-2]  = np.reshape(out_bias,[-1,1])\n",
    "  \n",
    "  if i+1 < len(string):\n",
    "    true_char = string[i+1]\n",
    "  else:\n",
    "    true_char = \"\"\n",
    "  loss = sess.run( [train_loss_np], feed_dict={output_placeholder: outpt_np, target_placeholder:y[i]})\n",
    "\n",
    "  d = {\"index\" : i, \"state\": h, \"input\":input_val, \"state_in\":h_in, \"delta\":h-h_in, \"loss\": loss, \"history\":history[:], \"output\": outpt_np, \"props\": soft_max(outpt_np), \"history_cont\": history_cont[:], \n",
    "       \"input_char\":string[i], \"predict_char\":chars[np.argmax(outpt_np)], \"true_char\": true_char }\n",
    "  df_limit_history = df_limit_history.append(d, ignore_index=True)\n",
    "\n",
    "\n",
    "\n",
    "fig = standard_figure()\n",
    "\n",
    "supercanvas = np.zeros((max_history, range_temp))\n",
    "superprob = np.zeros(max_history)\n",
    "#superprobcanvas = np.zeros((max_history,range_temp))\n",
    "\n",
    "for i in range(max_history):\n",
    "  outpt_np = np.zeros((1,27))\n",
    "  for k in range(range_temp):\n",
    "    nom_val = np.linalg.norm(canvas[i*28 : (i) * 28 + 27,(k)*2:(k*2+1)])\n",
    "    if nom_val >0:\n",
    "        supercanvas[i,k] = np.linalg.norm(canvas[i*28 : (i) * 28 + 27,(k)*2:(k*2+1)])\n",
    "    else:\n",
    "        supercanvas[i,k] = np.nan\n",
    "    if k <  range_temp -1:\n",
    "      outpt_np = outpt_np + canvas[i*28 : (i) * 28 + 27,(k)*2:(k*2+1)]\n",
    "#     if i < len(string) - 1:\n",
    "#       superprobcanvas[i, k ] = np.reshape( soft_max(outpt_np) ,[-1,1])[ mapping[string[i+1]]]\n",
    "  if i < len(string) - 1:\n",
    "    superprob[i] = probabilities[i*28 + mapping[string[i+1]]]\n",
    "  \n",
    "    \n",
    "#(ax1, ax2) = plt.subplots(2, sharex=True)  \n",
    "plt.subplot(2,1,1)    \n",
    "ax  = plt.gca()\n",
    "#plt.imshow(np.transpose(canvas), 'jet', interpolation = 'nearest', vmin = -5, vmax = 10)\n",
    "plt.imshow(np.transpose(supercanvas), 'jet', interpolation = 'nearest', vmin = -5, vmax = 10)\n",
    "#plt.ylabel(\"kappa per word\")\n",
    "plt.text(-6.5,+5, \"$||\\kappa^{t}_{word}||$\",fontsize = 17 )\n",
    "\n",
    "_=plt.xticks( np.arange(0,max_history,1) )\n",
    "_=plt.yticks( np.arange(-1,-1,max_history) )\n",
    "plt.axis('tight')\n",
    "\n",
    "for i in range(max_history):\n",
    "  val =  string[i]\n",
    "  if val ==\" \":\n",
    "    val = \"_\"\n",
    "  plt.text(i-0.2, -1.4, val)\n",
    "plt.text(-4, -1.4, 'Input:')\n",
    "\n",
    "#fig.colorbar(ax)\n",
    "\n",
    "plt.subplot(2,1,2, sharex=ax )  \n",
    "plt.ylabel(\"$P(\\mathbf{x}_{t+1}$)\", fontsize= 'medium')\n",
    "_ = plt.xticks( np.arange(-0.5,max_history,1),[])\n",
    "_ = plt.yticks( np.arange(0,1.1,0.5) )\n",
    "plt.ylim(0,1.0)\n",
    "\n",
    "for i in range(max_history):\n",
    "  if i < len(string) -1:\n",
    "    val =  string[i+1]\n",
    "    if val ==\" \":\n",
    "      val = \"_\"\n",
    "    plt.text(i-0.2, -0.15, val)\n",
    "\n",
    "for i in range(  superprobcanvas.shape[1] ):\n",
    "  _ = plt.plot( superprobcanvas[:-1,i],alpha = (1- i *1.0/superprobcanvas.shape[1]) , color = 'b' ) \n",
    "\n",
    "\n",
    "plt.savefig(save_dir + \"master_plot_caused_conflict.pdf\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "AyVjvIBzCale"
   },
   "outputs": [],
   "source": [
    "# SVD spectrum plot\n",
    "#\n",
    "\n",
    "standard_figure(width = 9, height = 4)\n",
    "\n",
    "train_string = ''.join([pretty_chars[x[i].argmax(1)[0]] for i in range(len(x))]).replace('_', ' ')\n",
    "hs = [h_zero.T]\n",
    "\n",
    "for char in train_string:\n",
    "  hs.append(make_step(hs[-1], mapping[char])[0])\n",
    "\n",
    "hs = np.hstack(hs[1:])\n",
    "\n",
    "\n",
    "u,s,v = np.linalg.svd(w_x[mapping['a']])\n",
    "\n",
    "v0_acts = v[:1,:].dot(hs)\n",
    "space_logs = (w_out.T.dot(hs) + out_bias[:,None])[mapping[' ']]\n",
    "\n",
    "plt.subplot(1,2,1)\n",
    "\n",
    "v0s = []\n",
    "\n",
    "for i,w in enumerate(w_x):\n",
    "  u,s,v = np.linalg.svd(w)\n",
    "  v0s.append(v[:1,:].T)\n",
    "  label=None\n",
    "  color='blue'\n",
    "  if i==0:\n",
    "    label='space'\n",
    "    color='red'\n",
    "  elif i==1:\n",
    "    label='a-z'\n",
    "  plt.plot(s, color=color, label=label)\n",
    "legend = plt.legend(loc='upper right', framealpha=1.)\n",
    "legend.get_frame().set_facecolor('#FFFFFF')\n",
    "legend.get_frame().set_alpha(1.0)\n",
    "plt.title('Singular value spectrum')\n",
    "plt.xlim(-10,225, )\n",
    "\n",
    "\n",
    "plt.subplot(1,2,2)\n",
    "space_logs = (w_out.T.dot(hs) + out_bias[:,None])[mapping[' ']]\n",
    "plt.scatter(v0_acts, space_logs, alpha=0.1, marker='.')\n",
    "#plt.axis('square')\n",
    "plt.xlabel(\"Hidden states projected onto\\nfirst singular vector of 'a'\")\n",
    "plt.ylabel(\"Unnormalized log prob.\\nof space token\")\n",
    "plt.title(\"First singular vectors predict space token\")\n",
    "plt.text(1, 20, \"$r=%.2f$\" % (np.corrcoef(v0_acts, space_logs)[0,1]),\n",
    "        bbox=dict(facecolor='white', alpha=0.5, edgecolor='none'))\n",
    "np.corrcoef(v0_acts, space_logs)\n",
    "# /plt.ylim(0, 1.2)\n",
    "\n",
    "\n",
    "plt.tight_layout()\n",
    "\n",
    "# save to png to not overload pdf readers with the pointcloud\n",
    "plt.savefig(save_dir + \"svd_spectrum.png\",dpi=300)\n",
    "\n",
    "#plt.savefig('./svd_spectrum.png',dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "n6Dw03f4Dt1l"
   },
   "outputs": [],
   "source": [
    "#Basis changes\n",
    "w_out_temp = np.transpose(w_out)\n",
    "u,s,v = np.linalg.svd(w_out_temp)\n",
    "temp = w_out_temp.dot(np.transpose(v))\n",
    "b_1_inv = np.transpose(v)\n",
    "b_1 = np.linalg.inv(b_1_inv)\n",
    "q_1 = w_out_temp.dot(b_1_inv)[:27,:27]\n",
    "\n",
    "block_1 = np.identity(num_nodes)\n",
    "for i in range(27):\n",
    "  q_1[i] = q_1[i] / np.linalg.norm(w_out_temp[i])\n",
    "block_1[:27,:27] = q_1\n",
    "\n",
    "plt.subplot(6,1,1)\n",
    "plt.imshow(w_out_temp.dot(b_1_inv))\n",
    "\n",
    "plt.subplot(6,1,2)\n",
    "readout_basis = b_1 #block_1.dot(b_1)\n",
    "plt.imshow(w_out_temp.dot(np.linalg.inv(readout_basis)))\n",
    "\n",
    "w_out_readout = np.transpose(w_out_temp.dot(np.linalg.inv(readout_basis)))\n",
    "w_out_pca = np.transpose(w_out_temp.dot(np.linalg.inv(pca_basis)))\n",
    "b_ixn_pca =  np.transpose(np.dot(pca_basis, np.transpose(b_ixn)))\n",
    "b_ixn_readout =  np.transpose(np.dot(readout_basis,  np.transpose(b_ixn)))\n",
    "#pca.components_w_out_readout = np.transpose(w_out_temp.dot(np.linalg.inv(readout_basis)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "VtQhuVZTD4hg"
   },
   "outputs": [],
   "source": [
    "### UNIGRAM / BIGRAM PLOTS\n",
    "#this cell builds the bigram_freq and cond_probs dicts. \n",
    "#bigram_freq[(char1, char2)] contains the counts of the bigram char1,char2 (both integers in [0,26]\n",
    "#cond_probs[(char1, char2)] contains prob(char2 | char1)\n",
    "\n",
    "\n",
    "freq = np.zeros(27)\n",
    "for i in x:\n",
    "  freq[np.argmax(i)] +=1\n",
    "  \n",
    "freq = freq / np.sum(freq)\n",
    "\n",
    "from collections import Counter\n",
    "bigram_freq = Counter()\n",
    "\n",
    "for i in xrange(len(x)-1):\n",
    "  bigram_freq[(np.argmax(x[i]), np.argmax(x[i+1]))] +=1\n",
    "\n",
    "cond_probs = Counter()\n",
    "\n",
    "def sum_cond(char_dict, char):\n",
    "  return np.sum([char_dict[(char, i)] for i in xrange(27)])\n",
    "\n",
    "for char in xrange(27):\n",
    "  total_count = sum_cond(bigram_freq, char)\n",
    "  for char2 in xrange(27):\n",
    "    cond_probs[(char, char2)] = float(bigram_freq[(char, char2)])/total_count\n",
    "  print char, total_count, sum_cond(cond_probs, char)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "n5cAQZSsDPcW"
   },
   "outputs": [],
   "source": [
    "#fig = standard_figure()\n",
    "seaborn.set_style(\"whitegrid\")\n",
    "#offsetnewbasis\n",
    "fig = plt.figure(figsize=(10, 3.5))\n",
    "\n",
    "norms = np.zeros(27)\n",
    "norms_readout_space = np.zeros(27)\n",
    "norms_orthogonal_space = np.zeros(27)\n",
    "\n",
    "for i in range(27):\n",
    "  #norms[i] =( np.linalg.norm(b_ixn[i]))\n",
    "  norms[i] =( np.linalg.norm(b_ixn_readout[i]))\n",
    "  norms_readout_space[i] = ( np.linalg.norm(b_ixn_readout[i][:27]))\n",
    "  norms_orthogonal_space[i] = ( np.linalg.norm(b_ixn_readout[i][27:]))\n",
    "  #print chars[ i ] , np.linalg.norm(b_h_pca[i])\n",
    "\n",
    "#w_out_readout\n",
    "#w_out_pca\n",
    "#b_ixn_pca\n",
    "#b_ixn_readout\n",
    "  \n",
    "plt.subplot(1,3,1)\n",
    "ax = plt.scatter(np.log(freq)/np.log(2),norms, c = np.arange(27), cmap = 'bwr', s = 45)\n",
    "plt.title(\"a)\")\n",
    "plt.xlabel(\"Bits of entropy\")\n",
    "plt.ylabel(r'$||\\mathbf{b}_\\mathbf{x}||$')\n",
    "\n",
    "for i in xrange(len(freq)):\n",
    "  plt.text(np.log(freq[i])/np.log(2)+0.2, norms[i], pretty_chars[i])\n",
    "\n",
    "plt.subplot(1,3,2)\n",
    "ax = plt.scatter(np.log(freq)/np.log(2),norms_readout_space, c = np.arange(27), cmap = 'bwr', s = 45)\n",
    "plt.title(\"b)\")\n",
    "plt.xlabel(\"Bits of entropy\")\n",
    "plt.ylabel(r\"$||\\mathbf{P}^{ro}_\\parallel \\mathbf{b}_\\mathbf{x}||$\")\n",
    "\n",
    "for i in xrange(len(freq)):\n",
    "  plt.text(np.log(freq[i])/np.log(2)+0.2, norms_readout_space[i], pretty_chars[i])\n",
    "\n",
    "plt.subplot(1,3,3)\n",
    "ax = plt.scatter(np.log(freq)/np.log(2),norms_orthogonal_space, c = np.arange(27), cmap = 'bwr', s = 45)\n",
    "plt.title(\"c)\")\n",
    "plt.xlabel(\"Bits of entropy\")\n",
    "plt.ylabel(r\"$||\\mathbf{P}^{ro}_\\perp \\mathbf{b}_\\mathbf{x}||$\")\n",
    "\n",
    "for i in xrange(len(freq)):\n",
    "  plt.text(np.log(freq[i])/np.log(2)+0.2, norms_orthogonal_space[i], pretty_chars[i])\n",
    "  \n",
    "#next_probs = np.array([[cond_probs[(i,j)] for j in xrange(27)] for i in xrange(27)])\n",
    "# mat2 = np.array([[kl_prob(get_cond_dist(i), get_cond_dist(j)) for j in xrange(27)] for i in xrange(27)])\n",
    "# mat2 = 1-np.array([[np.corrcoef(get_cond_dist(i), get_cond_dist(j))[0][1] for j in xrange(27)] for i in xrange(27)])\n",
    "# ax = plt.imshow(mat2, cmap = 'gray', interpolation = 'nearest')\n",
    "# plt.xticks(range(27), chars)\n",
    "# plt.yticks(range(27), chars)\n",
    "# plt.grid(False)\n",
    "# plt.subplot(1,3,3)\n",
    "\n",
    "# cos_dist = np.array([[sp.spatial.distance.cosine(b_ixn[i], b_ixn[j]) for j in xrange(27)] for i in xrange(27)])\n",
    "# plt.imshow(-cos_dist, cmap = 'gray', interpolation = 'none')\n",
    "# plt.xticks(range(27), chars)\n",
    "# plt.yticks(range(27), chars)\n",
    "# plt.grid(False)\n",
    "plt.tight_layout()\n",
    "\n",
    "\n",
    "\n",
    "# np.sum(mat2, axis = 0)\n",
    "plt.savefig(save_dir + \"norm_decay_broken_out.pdf\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "WOEVjUvnE4ND"
   },
   "outputs": [],
   "source": [
    "#fig = standard_figure()\n",
    "fig = plt.figure(figsize=(12, 4))\n",
    "\n",
    "order = ' aeoiu'\n",
    "letters_sorted = [' ','a','e','i','o', 'u']\n",
    "for l in chars:\n",
    "  if not l in order:\n",
    "    letters_sorted.append(l)\n",
    "index_func = []    \n",
    "for i,_ in enumerate(letters_sorted):\n",
    "  index_func.append( mapping[letters_sorted[i]])\n",
    "  \n",
    "    \n",
    "    \n",
    "\n",
    "components = [b_ixn_readout[index_func], b_ixn_readout[index_func,:27], b_ixn_readout[index_func,27: ] ]\n",
    "titles = ['a)', 'b)', 'c)' ]\n",
    "for ii in range(3):\n",
    "  plt.subplot(1,3,ii+1)\n",
    "  cos_dist = np.array([[sp.spatial.distance.cosine(components[ii][i], components[ii][j]) for j in xrange(27)] for i in xrange(27)])\n",
    "  plt.imshow(-cos_dist, cmap = 'gray', interpolation = 'nearest', vmin = -2, vmax = 0)\n",
    "  plt.xticks(range(27), letters_sorted)\n",
    "  plt.yticks(range(27), letters_sorted)\n",
    "  plt.title(titles[ii])\n",
    "  plt.grid(False)\n",
    "\n",
    "# plt.colorbar()\n",
    "  \n",
    "plt.savefig(save_dir + \"correlation_broken_out.pdf\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "xE6NA3OeFyLv"
   },
   "outputs": [],
   "source": [
    "\n",
    "w_x.shape\n",
    "b_ixn_corrected = np.zeros_like(b_ixn)\n",
    "for i in xrange(27):\n",
    "  b_corrected = np.dot(w_x[i], pca.mean_) + b_ixn[i]\n",
    "  b_ixn_corrected[i,:] = b_corrected\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "tnf9RedxFhQj"
   },
   "outputs": [],
   "source": [
    "print w_out.shape\n",
    "print b_ixn.shape\n",
    "print out_bias.shape\n",
    "\n",
    "chars_temp = chars[:]\n",
    "chars_temp[0] = \"_\"\n",
    "\n",
    "fig = standard_figure()\n",
    "plt.title(\"Decoded bias vs actual unigram probability\")\n",
    "val_x =soft_max(np.reshape(out_bias, [1,-1]))\n",
    "\n",
    "plt.subplot(1,3,1)\n",
    "_ = plt.plot(val_x[0], 'r', label = \"predicted\")\n",
    "\n",
    "ticks = range(0,26,5)\n",
    "\n",
    "plt.plot(freq,'g', label = 'empirical')\n",
    "#plt.title(\"P(char)\")\n",
    "plt.title(\"a)\")\n",
    "plt.xlim(0, 26)\n",
    "plt.xticks(ticks, [chars_temp[tick] for tick in ticks])\n",
    "plt.xlabel(\"Output Character\")\n",
    "plt.ylabel(\"Probability\")\n",
    "plt.legend()\n",
    "\n",
    "plt.subplot(1,3,2)\n",
    "#plot the conditional probs for a given char\n",
    "char = 0\n",
    "bias_output = np.transpose(np.dot(np.transpose(w_out), np.transpose(b_ixn))) + out_bias\n",
    "\n",
    "bias_output_pca = np.transpose(np.dot(np.transpose(w_out), np.transpose(b_ixn_corrected))) + out_bias\n",
    "plt.plot(soft_max(bias_output[char]), \"r\", label=\"predicted\")\n",
    "probs = [cond_probs[char, i] for i in xrange(27)]\n",
    "plt.plot(probs, \"g\", label = \"empirical\")\n",
    "plt.xlim(0, 26)\n",
    "\n",
    "plt.xticks(ticks, [chars_temp[tick] for tick in ticks])\n",
    "plt.xlabel(\"Output Character\")\n",
    "plt.ylabel(\"Conditional Probability\")\n",
    "#plt.title(\"P(char | space)\")\n",
    "plt.title(\"b)\")\n",
    "\n",
    "plt.legend(loc = 0)\n",
    "\n",
    "\n",
    "\n",
    "#compare correlation with bigram probs vs input basis probs\n",
    "corrs = []\n",
    "corrs_uni = []\n",
    "kl_bigrams = []\n",
    "kl_unigrams = []\n",
    "for char in xrange(27):\n",
    "  probs_network = soft_max(bias_output[char])\n",
    "  cond_p = [cond_probs[char,i] for i in xrange(27)]\n",
    "  corr = np.corrcoef(probs_network, cond_p)[0][1]\n",
    "  corr_unigram = np.corrcoef(cond_p, freq)[0][1]\n",
    "  kl_bigram = sp.stats.entropy(cond_p, probs_network)\n",
    "  kl_unigram = sp.stats.entropy(cond_p, freq)\n",
    "  #print corr, corr_unigram, kl_unigram, kl_bigram\n",
    "  corrs.append(corr)\n",
    "  corrs_uni.append(corr_unigram)\n",
    "  kl_bigrams.append(kl_bigram)\n",
    "  kl_unigrams.append(kl_unigram)\n",
    "  \n",
    "plt.subplot(1,3,3)\n",
    "ax = plt.scatter(corrs_uni, corrs, c = range(27), cmap = \"hsv\",  s = 45)\n",
    "\n",
    "\n",
    "for k in xrange(len(kl_unigrams)):\n",
    "  plt.text(corrs_uni[k]+.02, corrs[k], chars_temp[k], size=13)\n",
    "plt.xlim(-.08, 1.00001)\n",
    "plt.ylim(0, 1)\n",
    "plt.xlabel(\"corr(empirical, unigram)\")\n",
    "plt.ylabel(\"corr(empirical, predicted)\")\n",
    "#plt.title(\"Predicted vs Unigram\")\n",
    "plt.title(\"c)\")\n",
    "x_pts = np.linspace(0, 1, 100)\n",
    "print np.min(corrs_uni)\n",
    "plt.plot(x_pts, x_pts, \"r\")\n",
    "#cbar = plt.colorbar(ax)\n",
    "\n",
    "plt.tight_layout()\n",
    "# plt.tight_layout(pad=0.4, w_pad=0.5, h_pad=1.0)\n",
    "plt.savefig(save_dir + \"unigram_corr.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "cV3eRtZSDOdH"
   },
   "outputs": [],
   "source": [
    "fig = standard_figure()\n",
    "\n",
    "canvas1 = canvas[:, ::2]\n",
    "canvas1 = canvas1.reshape((-1, 28, canvas1.shape[-1]))\n",
    "final_lp = canvas1[:, :-1, -1]\n",
    "canvas1 = canvas1[:,:-1,:-1]\n",
    "unigram_lp = canvas1[0,:,-1]\n",
    "\n",
    "canvas2 = np.zeros((canvas1.shape[0], canvas1.shape[-1]-1))\n",
    "\n",
    "letters = np.zeros(canvas2.shape)\n",
    "\n",
    "  \n",
    "aggregate_low = lambda i,j: j #from step j\n",
    "aggregate_high = lambda i,j: j+1 #till step j\n",
    "#aggregate_high = lambda i,j: i+1 #till step i\n",
    "\n",
    "measure = 'norm'\n",
    "measure = 'xentropy'\n",
    "measure = 'rev_xentropy'\n",
    "measure = 'rev_kl'\n",
    "measure = 'kl'\n",
    "\n",
    "measure = 'entropy'\n",
    "\n",
    "\n",
    "for i in range(canvas1.shape[0]):\n",
    "  cell = canvas1[i, :, :]\n",
    "  kl_divs = np.zeros((cell.shape[-1]-1, ))\n",
    "  first_err = -1\n",
    "  for j in xrange(kl_divs.shape[0]):\n",
    "    cum_log_probs = unigram_lp + cell[:,aggregate_low(i,j):aggregate_high(i,j)].sum(1)\n",
    "    letters[i, j] = np.argmax(cum_log_probs)\n",
    "    if measure == \"kl\":\n",
    "      kl_divs[j] = kl(final_lp[i, :], cum_log_probs)\n",
    "    elif measure == \"rev_kl\":\n",
    "      kl_divs[j] = kl(cum_log_probs, final_lp[i, :])\n",
    "    elif measure == \"norm\":\n",
    "      kl_divs[j] = np.linalg.norm(cum_log_probs)\n",
    "    elif measure == \"entropy\":\n",
    "      kl_divs[j] = entropy(cum_log_probs)\n",
    "    elif measure == \"xentropy\":\n",
    "      kl_divs[j] = xentropy(final_lp[i, :], cum_log_probs)\n",
    "    elif measure == \"rev_xentropy\":\n",
    "      kl_divs[j] = xentropy(cum_log_probs, final_lp[i, :])\n",
    "    else:\n",
    "      raise Exception(\"Unknown measure\")\n",
    "  kl_divs[i+1:] = np.nan\n",
    "  canvas2[i, :] = kl_divs\n",
    "\n",
    "plt.imshow(canvas2.T, 'Blues_r', interpolation='nearest', vmax=4)\n",
    "plt.grid(False)\n",
    "plt.axis('tight')\n",
    "plt.xticks(*zip(*enumerate(string[1:])))\n",
    "plt.yticks(*zip(*list(enumerate(string))))\n",
    "\n",
    "if len(string)< 20:\n",
    "  for i,j in zip(*np.nonzero(np.isfinite(canvas2))):\n",
    "    if i==len(string)-1:\n",
    "      continue\n",
    "    if int(letters[i,j]) == mapping[string[i+1]]:\n",
    "      color='k'\n",
    "    else:\n",
    "      color='r'\n",
    "    plt.text(i,j+0.3,pretty_chars[int(letters[i,j])], color=color)\n",
    "plt.colorbar()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "i2jYRPV7F767"
   },
   "outputs": [],
   "source": [
    "#zoom in plot\n",
    "fig = plt.figure(figsize=(14, 6))\n",
    "\n",
    "\n",
    "p = plt.imshow(np.transpose(canvas[zoom_idx * 28: zoom_idx * 28 + 27, :]), 'jet', interpolation= 'nearest', vmin = -5, vmax = 10)\n",
    "plt.xticks(np.arange(0, 27), chars)\n",
    "label_chars = df_limit_history[\"input_char\"][:zoom_idx+1]\n",
    "for i in xrange(len(label_chars)):\n",
    "  if label_chars[i] == \" \":\n",
    "    label_chars[i] = \"''\"\n",
    "plt.yticks(2*np.arange(zoom_idx+1) + 0.5, label_chars)\n",
    "plt.xlabel(\"output character\")\n",
    "plt.ylabel(\"input character\")\n",
    "plt.title(\"Breakdown of Logit Contribution\")\n",
    "plt.grid(False)\n",
    "plt.colorbar(p)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "UFevl_OKptZk"
   },
   "outputs": [],
   "source": [
    "plt.plot(space_norms)\n",
    "plt.plot(first_letter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "OqniFa4UUUzH"
   },
   "outputs": [],
   "source": [
    "from collections import namedtuple\n",
    "Eigs = namedtuple('Eigs', ['e_vals', 'r_vecs', 'l_vecs', 'l_n_vecs'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "3pSy1QsruH56"
   },
   "outputs": [],
   "source": [
    "h_zero_expanded = np.hstack([h_zero, [[1]]])\n",
    "w_x_expanded = np.zeros(w_x.shape + np.array([0, 1, 1]))\n",
    "w_x_expanded[:,:-1,:-1] = w_x\n",
    "w_x_expanded[:,:-1,-1] = b_ixn\n",
    "w_o_expanded = np.zeros((27, 217))\n",
    "w_o_expanded[:,:-1] = w_out.T\n",
    "w_o_expanded[:,-1] = out_bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "eA-NbpykRhYm"
   },
   "outputs": [],
   "source": [
    "eigs = []\n",
    "\n",
    "for i, w in enumerate(w_x_expanded):\n",
    "  e_vals, r_vecs = np.linalg.eig(w,)\n",
    "  \n",
    "  order = np.argsort(-np.abs(e_vals))\n",
    "  e_vals = e_vals[order]\n",
    "  r_vecs = r_vecs[:, order]\n",
    "  \n",
    "  l_vecs = np.linalg.pinv(r_vecs)\n",
    "  \n",
    "  eigs.append(Eigs(e_vals=e_vals, r_vecs=r_vecs, l_vecs=l_vecs,\n",
    "                  l_n_vecs=l_vecs/np.sqrt((np.abs(l_vecs)**2).sum(1, keepdims=True))))\n",
    "  \n",
    "  #v0s.append(r_vecs[:1,:].T)\n",
    "  label=None\n",
    "  color='blue'\n",
    "  if i==0:\n",
    "    label='space'\n",
    "    color='red'\n",
    "  elif i==1:\n",
    "    label='a-z'\n",
    "  plt.plot(np.abs(e_vals), color=color, label=label)\n",
    "  \n",
    "#legend = plt.legend(loc='upper right', framealpha=1.)\n",
    "#legend.get_frame().set_facecolor('#FFFFFF')\n",
    "#legend.get_frame().set_alpha(1.0)\n",
    "#plt.title('Singular value spectrum')\n",
    "#plt.xlim(-10,225, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "vDyyLVxQ4p4o"
   },
   "outputs": [],
   "source": [
    "b_ixn.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "wFqAXrVp6MAi"
   },
   "outputs": [],
   "source": [
    "def get_word_eigs(word):\n",
    "  Ww1 = np.eye(217)\n",
    "  for character in word:\n",
    "    Ww1 = w_x_expanded[mapping[character]].dot(Ww1)\n",
    "\n",
    "  e_vals_w1, r_vecs_w1 = np.linalg.eig(Ww1)\n",
    "  order = np.argsort(-np.abs(e_vals_w1))\n",
    "  e_vals_w1 = e_vals_w1[order]\n",
    "  r_vecs_w1 = r_vecs_w1[:, order]\n",
    "\n",
    "  l_vecs_w1 = np.linalg.pinv(r_vecs_w1)\n",
    "  return Ww1, Eigs(e_vals=e_vals_w1, r_vecs=r_vecs_w1, l_vecs=l_vecs_w1,\n",
    "                   l_n_vecs=l_vecs_w1/np.sqrt((np.abs(l_vecs_w1)**2\n",
    "                                              ).sum(1, keepdims=True)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "ypwUhapE27Nf"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(15,15), dpi=150)\n",
    "#h0 = np.vstack((b_ixn[mapping[' ']][:,None], [[1]]))\n",
    "\n",
    "Ww1, eigs_w1 = get_word_eigs(' united')\n",
    "Ww2, eigs_w2 = get_word_eigs('states')\n",
    "\n",
    "eigs_s = eigs[mapping[' ']]\n",
    "lambda_s_sq = np.diag(np.sqrt(eigs_s.e_vals))\n",
    "\n",
    "LSR1 = lambda_s_sq.dot(eigs_s.l_vecs).dot(eigs_w1.r_vecs)\n",
    "L2RS = eigs_w2.l_vecs.dot(eigs_s.r_vecs.dot(lambda_s_sq))\n",
    "\n",
    "LSR1_power = (np.abs(LSR1)**2).sum(1)\n",
    "L2RS_power = (np.abs(L2RS)**2).sum(0)\n",
    "\n",
    "plt.subplot(2,2,1)\n",
    "plt.imshow(np.abs(LSR1), 'jet', interpolation='nearest')\n",
    "plt.colorbar()\n",
    "\n",
    "plt.subplot(2,2,2)\n",
    "plt.imshow(np.abs(L2RS), 'jet', interpolation='nearest')\n",
    "plt.colorbar()\n",
    "\n",
    "plt.subplot(2,2,3)\n",
    "img_overlap = np.zeros((217,217,3))\n",
    "img_overlap[:,:,0] = np.abs(LSR1) / np.max(np.abs(LSR1))\n",
    "img_overlap[:,:,1] = np.abs(L2RS) / np.max(np.abs(L2RS))\n",
    "plt.imshow(img_overlap, interpolation='nearest')\n",
    "\n",
    "#plt.xlim(0,25)\n",
    "#plt.ylim(25,0)\n",
    "\n",
    "plt.subplot(2,2,4)\n",
    "plt.semilogy(LSR1_power, label='LSR1_power')\n",
    "plt.semilogy(L2RS_power, label='L2RS_power')\n",
    "plt.legend()\n",
    "\n",
    "np.corrcoef(LSR1_power, L2RS_power)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "SCSX_x2nI8JM"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "UYTRcj2Zkiud"
   },
   "outputs": [],
   "source": [
    "def pseudso_ent(x):\n",
    "  x = np.abs(x)\n",
    "  x /= x.sum()\n",
    "  return -(np.log(x + 1e-20) * x).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "1S3hgBshcYZb"
   },
   "outputs": [],
   "source": [
    "keep_dim = 50\n",
    "res = np.zeros((27, 27))\n",
    "for i in xrange(27):\n",
    "  for j in xrange(27):\n",
    "    MM = eigs[j].l_n_vecs.dot(eigs[i].r_vecs)\n",
    "    res[i, j] = pseudo_ent(MM[:keep_dim, :keep_dim])\n",
    "    # plt.imshow(np.abs(MM),'jet')\n",
    "    # plt.colorbar()\n",
    "\n",
    "plt.imshow(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "MkYR4cXAlbSD"
   },
   "outputs": [],
   "source": [
    "res[np.arange(27),np.arange(27)] = np.nan\n",
    "plt.imshow(res, 'jet', interpolation='nearest')\n",
    "plt.colorbar()\n",
    "plt.xticks(*zip(*enumerate(pretty_chars)))\n",
    "plt.yticks(*zip(*enumerate(pretty_chars)))\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "NqHb8EWsyiwX"
   },
   "outputs": [],
   "source": [
    " np.diag(np.sqrt(eigs[mapping[bigram[1]]].e_vals)\n",
    "             ).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "BZ-QjqhZckIl"
   },
   "outputs": [],
   "source": [
    "bigram='th'\n",
    "sL = np.diag(np.sqrt(eigs[mapping[bigram[1]]].e_vals)\n",
    "             ).dot(eigs[mapping[bigram[1]]].l_vecs)\n",
    "sR = eigs[mapping[bigram[0]]].r_vecs.dot(\n",
    "    np.diag(np.sqrt(eigs[mapping[bigram[0]]].e_vals)))\n",
    "\n",
    "MM = sL.dot(sR)\n",
    "plt.imshow(np.abs(MM),'jet', interpolation='nearest')\n",
    "plt.colorbar()\n",
    "#plt.xlim(0,15)\n",
    "#plt.ylim(0,15)\n",
    "\n",
    "plt.figure()\n",
    "plt.hist(np.abs(MM.ravel()), 256)\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "Y7LQPY_3z1wd"
   },
   "outputs": [],
   "source": [
    "bigram='th'\n",
    "sL = np.diag(np.sqrt(eigs[mapping[bigram[1]]].e_vals)\n",
    "             ).dot(eigs[mapping[bigram[1]]].l_vecs)\n",
    "sR = eigs[mapping[bigram[0]]].r_vecs.dot(\n",
    "    np.diag(np.sqrt(eigs[mapping[bigram[0]]].e_vals)))\n",
    "\n",
    "MM = sL.dot(sR)\n",
    "plt.imshow(np.abs(MM),'jet', interpolation='nearest')\n",
    "plt.colorbar()\n",
    "#plt.xlim(0,15)\n",
    "#plt.ylim(0,15)\n",
    "\n",
    "plt.figure()\n",
    "plt.hist(np.abs(MM.ravel()), 256)\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "uqygjUPjhWjU"
   },
   "outputs": [],
   "source": [
    "MM.dot(MM1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "SIXlBiZgachC"
   },
   "outputs": [],
   "source": [
    "plt.scatter(np.real(e_vals), np.imag(e_vals))\n",
    "plt.axis('square')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "9xLypw4_UK1Q"
   },
   "outputs": [],
   "source": [
    "np.max(np.abs(l_vecs.dot(w) - np.diag(e_vals).dot(l_vecs)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "nuWunYRTXuNj"
   },
   "outputs": [],
   "source": [
    "train_string = ' annual revenue increased beyond expectations '\n",
    "hs = [h_zero.T]\n",
    "\n",
    "for char in train_string:\n",
    "  hs.append(make_step(hs[-1], mapping[char])[0])\n",
    "\n",
    "hs = np.hstack(hs[1:])\n",
    "\n",
    "space_logs = (w_out.T.dot(hs) + out_bias[:,None])[mapping[' ']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "uzJwc8yQZCIu"
   },
   "outputs": [],
   "source": [
    "plt.plot(space_logs)\n",
    "plt.plot(np.real(r_vecs[:, :1]).T.dot(hs).T)\n",
    "plt.xticks(*zip(*enumerate(train_string)))\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "FjJeI5LRWYX_"
   },
   "outputs": [],
   "source": [
    "#plt.plot(np.real(l_vecs[:1,:]).dot(hs).ravel())\n",
    "\n",
    "plt.scatter(np.real(l_vecs[:1,:]).dot(hs).ravel(), space_logs, alpha=0.1, marker='.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "TsYITLXseVW8"
   },
   "outputs": [],
   "source": [
    "b_ixn.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "A8RQCc4Mqv_Q"
   },
   "outputs": [],
   "source": [
    "probs = [cond_probs[(mapping['q'],i)] for i in xrange(27)]\n",
    "plt.plot(probs )\n",
    "plt.xticks(range(27), pretty_chars)\n",
    "None\n",
    "print entropy_probs(probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "l12yds2OzgvV"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "SPUmwqE7y2PK"
   },
   "outputs": [],
   "source": [
    "print np.dot(w_out.T, pca.mean_).shape\n",
    "np.linalg.norm(np.dot(w_out.T, pca.mean_))/np.linalg.norm(pca.mean_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "07YvZ7t8grof"
   },
   "outputs": [],
   "source": [
    "plt.plot(pca.mean_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "8UFSuF10iNhi"
   },
   "outputs": [],
   "source": [
    "bias_output_pca[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "Wh6AIKLLiREK"
   },
   "outputs": [],
   "source": [
    "bias_output[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "Je_L4DxHcMsy"
   },
   "outputs": [],
   "source": [
    "char = 5\n",
    "#bias_output_pca = np.transpose(np.dot(np.transpose(w_out), np.transpose(b_ixn_corrected))) + out_bias\n",
    "plt.plot(soft_max(bias_output[char]), \"b\", label=\"predicted\")\n",
    "plt.plot(soft_max(bias_output_pca[char]), \"r\", label=\"predicted_pca\")\n",
    "probs = [cond_probs[char, i] for i in xrange(27)]\n",
    "plt.plot(probs, \"g\", label = \"empirical\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "yFYhOoqKzilG"
   },
   "outputs": [],
   "source": [
    " \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "2G0WkJIBIMUZ"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "R5dQbhwWdRPa"
   },
   "outputs": [],
   "source": [
    "pca.mean_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "hDTIXTdeyQtf"
   },
   "outputs": [],
   "source": [
    "print i \n",
    "print j \n",
    "print df_limit_history[\"input_history\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "7SWXcGn1wgXL"
   },
   "outputs": [],
   "source": [
    "x_lim = 50\n",
    "#normalising = np.sum(( logit_val[:,:x_lim+1,i]  != 0), 0 )\n",
    "y_s =  (logit_val[:,:,i].sum(0)[:x_lim+1]) #/normalising\n",
    "x_s =  (vals[:,:,i].sum(0)[:x_lim+1]) #/normalising\n",
    "\n",
    "plt.plot(y_s / x_s)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "pAF4-QKGMIte"
   },
   "outputs": [],
   "source": [
    "#fig2 NORM OF ELEMENTS IN THE BUFFER\n",
    "errors = np.zeros(50)\n",
    "#h  = np.zeros((1, num_nodes))\n",
    "h  = h_zero\n",
    "losses = []\n",
    "counter = 0\n",
    "max_history = 200\n",
    "history = []\n",
    "input_history = [0 for i in xrange(max_history)]\n",
    "#df_limit_history = pd.DataFrame()\n",
    "\n",
    "count = 0\n",
    "for i in range(0,60000):\n",
    "  if i % 1000 == 0:\n",
    "    print i, len(df_limit_history)\n",
    "  h_in = h\n",
    "  input_val = np.argmax(x[i])\n",
    "  for k in range(min(max_history, count)):\n",
    "    new_h = np.dot( w_x[input_val], np.reshape( history[k], [-1,1] ))\n",
    "    out_val = out_val + new_h\n",
    "    history[k] = new_h\n",
    "    \n",
    "  out_val = np.zeros((num_nodes,1))\n",
    "  for k in range(min(max_history, count)):\n",
    "    out_val = out_val +  history[k]\n",
    "    \n",
    "  h = out_val +  np.reshape(b_ixn[input_val], [-1,1] ) \n",
    "  if len( history) == max_history: \n",
    "    history[-1] = history[-1] + history[-2]\n",
    "    history.pop(-2)\n",
    "    input_history.pop(-2)\n",
    "  history.insert(0,np.reshape(b_ixn[input_val], [-1,1] ) )\n",
    "  input_history.insert(0,input_val)\n",
    "  outpt_np = np.dot( np.reshape(h, [1,-1]), w_out) + out_bias\n",
    "  \n",
    "  #outpt_np = np.dot( np.reshape(h, [1,-1]), w_out) + out_bias\n",
    "  loss = sess.run( [train_loss_np], feed_dict={output_placeholder: outpt_np, target_placeholder:y[i]})\n",
    "  outpt_np_new = outpt_np\n",
    "  for t in range(len(history)-1, -1,-1):\n",
    "    outpt_np = outpt_np - np.dot( np.reshape(history[t], [1,-1]), w_out)[0]  \n",
    "    if np.argmax( outpt_np_new ) != np.argmax(outpt_np):\n",
    "      #print t, input_history[t]\n",
    "      d = {\"index\" : i, \"input\":np.argmax(x[i]), \"distance\": t, \"trigger_offset\": input_history[t]}\n",
    "      df_limit_history = df_limit_history.append(d, ignore_index=True)\n",
    "      break\n",
    "    if t == 0:\n",
    "      d = {\"index\" : i, \"input\":np.argmax(x[i]), \"distance\": t, \"trigger_offset\": input_history[t]}\n",
    "      df_limit_history = df_limit_history.append(d, ignore_index=True)\n",
    "  count += 1\n",
    "  if i  > 10:\n",
    "    losses.append(loss)\n",
    "#     d = {\"index\" : i, \"state\": h, \"input\":np.argmax(x[i]), \"state_in\":h_in, \"delta\":h-h_in, \"dist\": i % 50, \"loss\": loss, \"history\":history[:], \"input_history\": input_history[:] }\n",
    "#     df_limit_history = df_limit_history.append(d, ignore_index=True)\n",
    "print(\"produced {} random states\".format(n_random))\n",
    "print(np.mean(losses))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "Vt64u9xUPdDH"
   },
   "outputs": [],
   "source": [
    "fig = standard_figure()\n",
    "\n",
    "# plt.subplot(1,2,1)\n",
    "# _= plt.hist(df_limit_history[\"distance\"].values,bins= 100)\n",
    "\n",
    "mean_by_char = df_limit_history.groupby(['trigger_offset',\"input\"])[\"distance\"].mean()\n",
    "count = df_limit_history.groupby(['trigger_offset',\"input\"])[\"distance\"].count()\n",
    "# without_space = df_limit_history_space.groupby(['counter'])[\"loss_other\"].median().values\n",
    "# all_chars = df_limit_history_space.groupby(['counter'])[\"loss\"].median().values\n",
    "\n",
    "canvas = np.zeros((27,27))*np.nan\n",
    "canvas_count = np.zeros((27,27))*np.nan\n",
    "\n",
    "for i in range(27):\n",
    "  for j in range(27):\n",
    "    temp = df_limit_history[ (df_limit_history[\"trigger_offset\"] == j) & (df_limit_history[\"input\"] == i) & (df_limit_history[\"distance\"] >0 )]\n",
    "    canvas_count[i,j] = len(temp[\"distance\"]) \n",
    "\n",
    "    if len(temp[\"distance\"]) > 10:\n",
    "      canvas[i,j] = (temp[\"distance\"].median())\n",
    "ax1 = plt.subplot(1,2,1)\n",
    "ax = plt.imshow(canvas, cmap = 'jet', interpolation = 'nearest')\n",
    "plt.xticks(*zip(*enumerate(pretty_chars[:])))\n",
    "plt.yticks(*zip(*list(enumerate(pretty_chars))))\n",
    "ax1.yaxis.grid(False)\n",
    "ax1.xaxis.grid(False)\n",
    "fig.colorbar(ax)\n",
    "\n",
    "ax1 = plt.subplot(1,2,2)\n",
    "ax = plt.imshow(canvas_count, cmap = 'jet', interpolation = 'nearest')\n",
    "plt.xticks(*zip(*enumerate(pretty_chars[:])))\n",
    "plt.yticks(*zip(*list(enumerate(pretty_chars))))\n",
    "ax1.yaxis.grid(False)\n",
    "ax1.xaxis.grid(False)\n",
    "\n",
    "fig.colorbar(ax)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "zfA3Sd7pXPvk"
   },
   "outputs": [],
   "source": [
    "\n",
    "plt.plot(100*pca.explained_variance_ratio_, 'b')\n",
    "plt.title(\"Explained Variance Ratio\")\n",
    "plt.xlabel(\"Number of PCA dimension\")\n",
    "plt.ylabel(\"%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "dNyTVyL9i2yp"
   },
   "outputs": [],
   "source": [
    "norms = np.zeros(27)\n",
    "for i in range(0,27):\n",
    "  norms[i] = (np.linalg.norm(b_ixn[i]))\n",
    "  \n",
    "print norms.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "eM4dLxw9AW_L"
   },
   "outputs": [],
   "source": [
    "np.linalg.norm(b_ixn[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "7_A2wgga_vIr"
   },
   "outputs": [],
   "source": [
    "#  #soft_max(sum_of_hm1_cont_super + out_bias).shape\n",
    "# plt.imshow(np.transpose(superprobcanvas))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "hSwOF4prvqlR"
   },
   "outputs": [],
   "source": [
    "fig = standard_figure()\n",
    "\n",
    "canvas1 = canvas[:, ::2]\n",
    "canvas1 = canvas1.reshape((-1, 28, canvas1.shape[-1]))\n",
    "final_lp = canvas1[:, :-1, -1]\n",
    "canvas1 = canvas1[:,:-1,:-1]\n",
    "unigram_lp = canvas1[0,:,-1]\n",
    "\n",
    "canvas2 = np.zeros((canvas1.shape[0], canvas1.shape[-1]-1))\n",
    "\n",
    "letters = np.zeros(canvas2.shape)\n",
    "\n",
    "  \n",
    "aggregate_low = lambda i,j: j #from step j\n",
    "aggregate_high = lambda i,j: j+1 #till step j\n",
    "#aggregate_high = lambda i,j: i+1 #till step i\n",
    "\n",
    "measure = 'norm'\n",
    "\n",
    "measure = 'xentropy'\n",
    "measure = 'rev_xentropy'\n",
    "measure = 'rev_kl'\n",
    "measure = 'kl'\n",
    "\n",
    "measure = 'norm'\n",
    "measure = 'entropy'\n",
    "\n",
    "spc_count = 0\n",
    "\n",
    "for i in range(canvas1.shape[0]):\n",
    "  spc_count += string[i] == ' '\n",
    "  cell = canvas1[i, :, :]\n",
    "  kl_divs = np.zeros((cell.shape[-1]-1, ))\n",
    "  first_err = -1\n",
    "  for j in xrange(kl_divs.shape[0]):\n",
    "    cum_log_probs = unigram_lp + cell[:,aggregate_low(i,j):aggregate_high(i,j)].sum(1)\n",
    "    letters[i, j] = np.argmax(cum_log_probs)\n",
    "    if measure == \"kl\":\n",
    "      kl_divs[j] = kl(final_lp[i, :], cum_log_probs)\n",
    "    elif measure == \"rev_kl\":\n",
    "      kl_divs[j] = kl(cum_log_probs, final_lp[i, :])\n",
    "    elif measure == \"norm\":\n",
    "      kl_divs[j] = np.linalg.norm(cum_log_probs)\n",
    "    elif measure == \"entropy\":\n",
    "      kl_divs[j] = entropy(cum_log_probs)\n",
    "    elif measure == \"xentropy\":\n",
    "      kl_divs[j] = xentropy(final_lp[i, :], cum_log_probs)\n",
    "    elif measure == \"rev_xentropy\":\n",
    "      kl_divs[j] = xentropy(cum_log_probs, final_lp[i, :])\n",
    "    else:\n",
    "      raise Exception(\"Unknown measure\")\n",
    "  kl_divs[spc_count:] = np.nan\n",
    "  canvas2[i, :] = kl_divs\n",
    "\n",
    "plt.imshow(canvas2.T, 'Blues_r', interpolation='nearest')\n",
    "plt.grid(False)\n",
    "plt.axis('tight')\n",
    "plt.xticks(*zip(*enumerate(string[1:])))\n",
    "plt.yticks(*zip(*list(enumerate(string.strip().split()))))\n",
    "\n",
    "spcs = np.zeros((len(string),), dtype=\"|S1\")\n",
    "spcs[:] = list(string)\n",
    "spcs = np.nonzero(spcs==' ')[0]\n",
    "spcs = np.stack([spcs-1.5, spcs-0.5]).reshape(-1)\n",
    "\n",
    "import matplotlib.ticker\n",
    "plt.gca().xaxis.set_minor_locator(matplotlib.ticker.FixedLocator(spcs))\n",
    "plt.gca().yaxis.set_minor_locator(matplotlib.ticker.FixedLocator(np.arange(canvas2.shape[1])-0.5))\n",
    "\n",
    "plt.grid(which='minor')\n",
    "\n",
    "if len(string)< 20:\n",
    "  for i,j in zip(*np.nonzero(np.isfinite(canvas2))):\n",
    "    if i==len(string)-1:\n",
    "      continue\n",
    "    if int(letters[i,j]) == mapping[string[i+1]]:\n",
    "      color='k'\n",
    "    else:\n",
    "      color='r'\n",
    "    plt.text(i,j+0.3,pretty_chars[int(letters[i,j])], color=color)\n",
    "plt.colorbar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "oycs2eXQI5f4"
   },
   "outputs": [],
   "source": [
    "superprobcanvas[i, k ] = \n",
    "np.reshape( soft_max(outpt_np) ,[-1,1])[ mapping[string[i+1]]]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "XqciPept-6ow"
   },
   "outputs": [],
   "source": [
    "def predict(input_string, n, t, h = None, on_input = False, random_inputs = False, transform = None):\n",
    "  indx = range(len(x)-(n + len(input_string)))\n",
    "  np.random.shuffle(indx)\n",
    "  string = input_string\n",
    "  values = np.zeros((len(input_string) + n, num_nodes))\n",
    "  all_inputs = np.zeros((len(input_string) + n, num_nodes))\n",
    "  surprise = np.zeros((len(input_string) + n+1))\n",
    "  certainty = np.zeros((len(input_string)+ n))\n",
    "  input_vals = np.zeros((len(input_string) + n))\n",
    "  if h is None:\n",
    "    h  = h_zero\n",
    "  else:\n",
    "    h = np.reshape(h, [1,-1])\n",
    "  counter = 0\n",
    "  for i in range(len(input_string) + n):\n",
    "    h_in = h\n",
    "    if on_input:\n",
    "      if random_inputs or random_inputs:\n",
    "          inputs_t = x[indx[i]]\n",
    "      else:\n",
    "          inputs_t = x[i+indx[0]]\n",
    "      string += chars[np.argmax(inputs_t)]\n",
    "    else:\n",
    "      inputs_t = np.zeros((1,27))\n",
    "      if i < len(input_string):\n",
    "        inputs_t[0, mapping[ input_string[i]]] = 1\n",
    "      else:\n",
    "        inputs_t[0,choice] = 1\n",
    "        string = string + (char_out)\n",
    "    h, output_now = make_step(h, np.argmax(inputs_t))\n",
    "\n",
    "    #h,output_now, probs = sess.run( [hidden_state_source_pca, output_source_pca, output_eval_pca], feed_dict={temperature:[t]})\n",
    "    probs  = soft_max(output_now, t)\n",
    "    choice = np.random.choice(len(probs[0,:]), p=probs[0])\n",
    "    char_out = chars[choice] \n",
    "    values[i] = np.transpose(h)\n",
    "    all_inputs[i] = b_ixn[np.argmax(inputs_t)]\n",
    "    input_vals[i] = np.argmax(inputs_t)\n",
    "    certainty[i] = np.sum( probs[0,:]*np.log(probs[0,:])/np.log(2))\n",
    "    if on_input:\n",
    "      surprise[i+1] = np.log(probs[0,np.argmax(y[i])])/np.log(2)\n",
    "    else:\n",
    "      if i < len(input_string)-1: \n",
    "        surprise[i+1] = np.log(probs[0,mapping[input_string[i+1]]])/np.log(2)\n",
    "      else:\n",
    "        surprise[i+1] = np.log(probs[0,choice])/np.log(2)\n",
    "  if not (transform is None ):\n",
    "    values = np.transpose( transform.dot(np.transpose(values)))\n",
    "    all_inputs = np.transpose(transform.dot(np.transpose(all_inputs)))\n",
    "    print \"did transform\"\n",
    "\n",
    "  return values, output_now, string, all_inputs, -surprise, input_vals, certainty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "N2DStWBRHGVR"
   },
   "outputs": [],
   "source": [
    "#fig2\n",
    "#Barebones running of the RNN with all elements accessible in numpy. Also explores a limited buffer of linearly independent propagated offsets\n",
    "errors = np.zeros(50)\n",
    "#h  = np.zeros((1, num_nodes))\n",
    "h  = h_zero #sess.run(source_rnn_cell_pca.zero_state(1, dtype=tf.float32))\n",
    "\n",
    "counter = 0\n",
    "results = []\n",
    "for  max_history in range(0,20):\n",
    "    losses = []\n",
    "    history = [0 for i in xrange(max_history)]\n",
    "\n",
    "    df_limit_history = pd.DataFrame()\n",
    "\n",
    "    # w_x, b_ixn, w_out, h_zero, out_bias\n",
    "    count = 0\n",
    "    for i in range(10000, 20000):\n",
    "      if i % 1000 == 0:\n",
    "        print i, len(df_limit_history)\n",
    "    #   if i % 50 == 0:\n",
    "    #     h  = sess.run(source_rnn_cell_pca.zero_state(1, dtype=tf.float32))\n",
    "      h_in = h\n",
    "      input_val = np.argmax(x[i])\n",
    "      for k in range(min(max_history, count)):\n",
    "        new_h = np.dot( w_x[input_val], np.reshape( history[k], [-1,1] ))\n",
    "        out_val = out_val + new_h\n",
    "        history[k] = new_h\n",
    "\n",
    "      out_val = np.zeros((num_nodes,1))\n",
    "      for k in range(min(max_history, count)):\n",
    "        if not k > max_history:\n",
    "          out_val = out_val +  history[k]\n",
    "      h = out_val +  np.reshape(b_ixn[input_val], [-1,1] ) \n",
    "      if len( history) == max_history and False: \n",
    "        history[-1] = history[-1] + history[-2]\n",
    "        history.pop(-2)\n",
    "      if len( history) == max_history and True and max_history>0 : \n",
    "        #history[-1] = history[-1] + history[-2]\n",
    "        history.pop(-1)\n",
    "      history.insert(0,np.reshape(b_ixn[input_val], [-1,1] ) )\n",
    "\n",
    "      outpt_np = np.dot( np.reshape(h, [1,-1]), w_out) + out_bias\n",
    "      loss = sess.run( [train_loss_np], feed_dict={output_placeholder: outpt_np, target_placeholder:y[i]})\n",
    "      count += 1\n",
    "      if i  > 10:\n",
    "        losses.append(loss)\n",
    "        d = {\"index\" : i, \"state\": h, \"input\":np.argmax(x[i]), \"state_in\":h_in, \"delta\":h-h_in, \"dist\": i % 50, \"loss\": loss, \"history\":history[:] }\n",
    "        df_limit_history = df_limit_history.append(d, ignore_index=True)\n",
    "    print(\"produced {} random states\".format(n_random))\n",
    "    print(np.mean(losses))\n",
    "    results.append(np.mean(losses))\n",
    "\n",
    "    # #plt.plot(results)\n",
    "# #plt.show()\n",
    "\n",
    "#droppping the most recent offset from the current prediction reduces accuracy to 1.9, droppping the 1st contribution to 1.718, 2nd 1.66771\n",
    "fig = plt.figure(figsize=(10,10))\n",
    "\n",
    "plt.plot(results)\n",
    "plt.title(\"Cross-entropy vs length context\")\n",
    "\n",
    "plt.xlabel(\"Length of context\")\n",
    "plt.ylabel(\"Cross-entropy\")\n",
    "\n",
    "# plt.show()\n",
    "plt.savefig(save_dir + \"n_gram_plot.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "JEHY_kGuLlIY"
   },
   "outputs": [],
   "source": [
    "values, output_now, string, all_inputs, surprise, input_vals, certainty = predict(\" a lot \", 100, 2)\n",
    "\n",
    "values, output_now, string_annual, all_inputs, surprise, input_vals, certainty = predict(\" show me ing\", 100, 20)\n",
    "print string\n",
    "print string_annual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "mkIgD2VO_FA7"
   },
   "outputs": [],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "BLcQktdQyjX9"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "4v83Kr8x_tyc"
   },
   "outputs": [],
   "source": [
    "for word in words:\n",
    "  word_counts[word]+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "R6umvk0o_3ET"
   },
   "outputs": [],
   "source": [
    "word_counts.most_common()[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "jjvLPVHop7Ae"
   },
   "outputs": [],
   "source": [
    "def get_word_vec(word):\n",
    "  vec = b_ixn[0]\n",
    "  for c in word:\n",
    "    vec = np.dot(w_x[mapping[c]], vec)\n",
    "  return vec\n",
    "\n",
    "def get_space_norm(word, char = ' '):\n",
    "  vec = get_word_vec(word)\n",
    "  image = np.dot(w_x[mapping[char]], vec)\n",
    "  \n",
    "  return np.linalg.norm(image)/np.linalg.norm(vec)\n",
    "\n",
    "word_tups = []\n",
    "summary_df = pd.DataFrame()\n",
    "for word, count in word_counts.most_common():\n",
    "  if i %10000 == 0:\n",
    "    print i, len(summary_df)\n",
    "\n",
    "  d = {\"word\":word, \"shrinkage\":get_space_norm(word, ' '), \"count\":count}\n",
    "  summary_df = summary_df.append(d, ignore_index=True)\n",
    "\n",
    "print len(summary_df)\n",
    "\n",
    "summary_df = summary_df.drop_duplicates()\n",
    "print len(summary_df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "nRhlhafY1-j4"
   },
   "outputs": [],
   "source": [
    "plt.plot(summary_df[\"count\"], summary_df[\"shrinkage\"], 'r.')\n",
    "plt.xlim(0, 2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "EHplh9g8wLjO"
   },
   "outputs": [],
   "source": [
    "word_tups_sorted[-40:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "fk1eDP4GxwcF"
   },
   "outputs": [],
   "source": [
    "word_tups_sorted[:40]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "aFkZdGvGvl9i"
   },
   "outputs": [],
   "source": [
    "all_state, _, next_val, input_val, suprise, input_indx, certainty = predict(\"this is the life of someone who is living on the edg\", 50, 10, on_input = False, random_inputs = False) #basis\n",
    "fig = plt.figure(figsize=(28, 12))\n",
    "print(next_val)\n",
    "plt.subplot(2,4,1)\n",
    "_ = plt.plot(all_state[:,:50])\n",
    "plt.title(\"first 50 dimensions plotted as function of time\")\n",
    "plt.subplot(2,4,2)\n",
    "_ = plt.plot(np.transpose(all_state))\n",
    "plt.title(\"all dimensions plotted each line is one time step\")\n",
    "\n",
    "ax1 = plt.subplot(2,4,3)\n",
    "ax1.plot((np.sum(all_state**2,1)**0.5))\n",
    "plt.title(\"blue: norm of activation, red: certainty\")\n",
    "\n",
    "suprise = suprise\n",
    "ax2 = ax1.twinx()\n",
    "\n",
    "#ax2.plot(suprise, 'r')\n",
    "ax2.plot(certainty, 'r')\n",
    "\n",
    "plt.subplot(2,4,4)\n",
    "_ = plt.plot(np.log((np.mean(all_state**2,0))))\n",
    "plt.title(\"log of squared activations by dimension\")\n",
    "\n",
    "\n",
    "plt.subplot(2,4,5)\n",
    "plt.title(\"activations over time\")\n",
    "\n",
    "_  = plt.imshow(np.transpose(all_state[:,:100]),interpolation = 'none',cmap ='bwr')\n",
    "\n",
    "plt.subplot(2,4,6)\n",
    "plt.title(\"input offsets over time\")\n",
    "\n",
    "ax = plt.imshow(np.transpose(input_val[:,:100]),interpolation = 'none',cmap ='bwr')\n",
    "\n",
    "print(np.corrcoef(suprise[:-1], (np.sum(all_state**2,1)**0.5)))\n",
    "print(np.corrcoef(certainty, (np.sum(all_state**2,1)**0.5)))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "plt.subplot(2,4,7)\n",
    "plt.title(\"state - input over time\")\n",
    "\n",
    "plt.title(\"\")\n",
    "\n",
    "ax = plt.imshow(np.transpose(all_state[:,:100]-input_val[:,:100]),interpolation = 'none',cmap ='bwr')\n",
    "\n",
    "#ax = plt.scatter(suprise[:-1],  np.sum(all_state[:,:]**2,1), c = input_indx, cmap ='bwr')\n",
    "ax1 = plt.subplot(2,4,8)\n",
    "delta_norm = (np.sum((all_state[0:-1] -all_state[1:])**2,1))\n",
    "ax1.plot(delta_norm)\n",
    "\n",
    "print(np.corrcoef(suprise[:-2], delta_norm))\n",
    "\n",
    "\n",
    "ax2 = ax1.twinx()\n",
    "\n",
    "ax2.plot(suprise, 'r')\n",
    "plt.title(\"blue: change of activation, red: is surprise\")\n",
    "\n",
    "#ax2.plot(certainty, 'r')\n",
    "\n",
    "plt.savefig(\"/tmp/comprehensive_view.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "8zKpDps0wUK9"
   },
   "outputs": [],
   "source": [
    "#power_plot\n",
    "\n",
    "losses = []\n",
    "count = 0\n",
    "h  = h_zero\n",
    "counter = 0\n",
    "h_space = h\n",
    "h_other = h\n",
    "random_states = []\n",
    "losses = []\n",
    "split = np.zeros((5000,2))\n",
    "inputs = []\n",
    "inputs_num = []\n",
    "counter = 0\n",
    "df_temp = pd.DataFrame()\n",
    "for i in range(5000):\n",
    "  h_in = h\n",
    "  input_val = np.argmax(x[i])\n",
    "  if input_val ==0:\n",
    "    counter = 0\n",
    "  else:\n",
    "    counter = counter + 1\n",
    "  h, outpt_np, loss =  make_step(h, input_val, y[i])\n",
    "  h_basis_new = np.transpose(np.dot(b_1,  (h)))\n",
    "  split[i,0] = np.linalg.norm( h_basis_new )**2\n",
    "  split[i,1]= np.linalg.norm( h_basis_new[0][:27])**2\n",
    "  #split[i,1]= np.linalg.norm( h_basis_new[0][27:])/(216-27)\n",
    "  inputs.append(pretty_chars[input_val])\n",
    "  inputs_num.append(counter)\n",
    "  d = {\"input_val\":counter, \"length_one\":np.linalg.norm( h_basis_new )**2 ,  \"length_one\":np.linalg.norm( h_basis_new )**2\n",
    "\n",
    "\n",
    "  \n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "_lzbhg-YGm7i"
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(10,10))\n",
    "\n",
    "# plt.plot(split[:,1])\n",
    "# plt.plot(split[:,0])\n",
    "# plt.xticks(*zip(*list(enumerate(inputs))))\n",
    "# None\n",
    "\n",
    "# # #abs((b_1.T.dot(b_1)-np.eye(216))).sum()\n",
    "# # # if split[:,0] > 0:\n",
    "# # #   x_ax = split[:,1]/split[:,0]\n",
    "# # # else:\n",
    "# # #   x_ax \n",
    "# # plt.subplot(1,2,2)\n",
    "# # plt.scatter(x_ax[1:], np.array( inputs_num[0:-1]))\n",
    "# # #readout_basis\n",
    "# # #print x_ax[:5]\n",
    "# # #print inputs_num[:5]\n",
    "\n",
    "# #plt.plot(x_ax[:50],inputs_num[:50], '.')\n",
    "#plt.scatter(np.ones(10), np.zeros(10))\n",
    "plt.scatter(np.array( inputs_num[0:]), x_ax[:], )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#eigen-plot Eigen Spectrum of frequent n-grams vs random permutations of letters\n",
    "def get_word_vec(word):\n",
    "  vec = b_ixn[0]\n",
    "  for c in word:\n",
    "    vec = np.dot(w_x[mapping[c]], vec)\n",
    "  return vec\n",
    "def get_word_matrix(word):\n",
    "  m = np.eye(216)\n",
    "  for c in word:\n",
    "    m = np.dot(w_x[mapping[c]], m)\n",
    "  return m\n",
    "def get_word_evals(word, norm = False):\n",
    "  m = get_word_matrix(word)\n",
    "  e_vals, r_vecs = np.linalg.eig(m)\n",
    "  if norm:\n",
    "    e_vals = [np.absolute(e) for e in e_vals]\n",
    "  return e_vals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vals = get_word_evals(\"unite\")\n",
    "vals_norm = [np.absolute(v) for v in vals]\n",
    "print np.mean(vals_norm), np.max(vals_norm)\n",
    "\n",
    "r = [v.real for v in vals]\n",
    "c = [v.imag for v in vals]\n",
    "p = plt.plot(r, c, 'r.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#get all N-grams from the test data\n",
    "all_text = \"\".join([chars[np.argmax(t)] for t in y])\n",
    "\n",
    "N = 4\n",
    "n_grams = [all_text[i:i+N] for i in xrange(0, len(all_text)-N-1)]\n",
    "n_grams_no_space = [word for word in n_grams if word.find(\" \") == -1]\n",
    "print len(n_grams_no_space)\n",
    "print len(n_grams)\n",
    "print n_grams_no_space[5]\n",
    "counts = pd.value_counts(n_grams_no_space)\n",
    "print counts.index[:10] #most common n-grams not containing a space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#plot max and mean norm eigenvalue for the top 100 n-grams\n",
    "K = 100\n",
    "top_ngrams = counts.index[:K]\n",
    "max_eigs = []\n",
    "mean_eigs = []\n",
    "for gram in top_ngrams:\n",
    "  e_vals = get_word_evals(gram, norm=True)\n",
    "  max_eigs.append(np.max(e_vals))\n",
    "  mean_eigs.append(np.mean(e_vals))\n",
    "\n",
    "plt.subplot(2,1,1)\n",
    "p = plt.hist(max_eigs, bins=20)\n",
    "\n",
    "plt.subplot(2,1,2)\n",
    "p = plt.hist(mean_eigs, bins=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#same plot but take a random permutation of each ngram\n",
    "K = 100\n",
    "top_ngrams = counts.index[:K]\n",
    "max_eigs = []\n",
    "mean_eigs = []\n",
    "for gram in top_ngrams:\n",
    "  gram2 = ''.join(random.sample(gram,len(gram))) #permute!\n",
    "  #print gram, gram2\n",
    "  e_vals = get_word_evals(gram2, norm=True)\n",
    "  max_eigs.append(np.max(e_vals))\n",
    "  mean_eigs.append(np.mean(e_vals))\n",
    "\n",
    "plt.subplot(2,1,1)\n",
    "p = plt.hist(max_eigs, bins=20)\n",
    "\n",
    "plt.subplot(2,1,2)\n",
    "p = plt.hist(mean_eigs, bins=20)\n",
    "\n",
    "#end eigen-plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "xBzw_cqeMPw3"
   },
   "outputs": [],
   "source": [
    "\n",
    "#Figure:BasisTransform pca_and_readout\n",
    "fig = standard_figure(5)\n",
    "\n",
    "matrices = [w_out, b_ixn, w_out_pca, b_ixn_pca, w_out_readout, b_ixn_readout] \n",
    "\n",
    "plot_pre_fix = [\"Readout \", \"Biases \"]\n",
    "basis_label = [\"Original\", \"PCA\" , \"Readout\" ]\n",
    "basises = [np.eye(num_nodes), pca_basis, readout_basis]\n",
    "\n",
    "\n",
    "counter = 0\n",
    "#import matplotlib.gridspec as gridspec\n",
    "#gs = gridspec.GridSpec(2, 2)\n",
    "#ax = plt.subplot(gs[0, 0])\n",
    "\n",
    "for row in range(3):\n",
    "  for col in range(2):\n",
    "    if col == 0:\n",
    "      matrix = np.transpose(matrices[counter])\n",
    "    else:\n",
    "      matrix = matrices[counter]\n",
    "      \n",
    "    title = plot_pre_fix[col] + \" Matrix in \"+ basis_label[row] + \" Basis\"\n",
    "    \n",
    "    #ax = plt.subplot2grid((3,7), (row, col*3), colspan=3) #plt.subplot(3,2,counter + 1)\n",
    "    ax = plt.subplot(3,2,counter + 1)\n",
    "    plt.title(title)\n",
    "    plt.imshow(matrix,interpolation = 'nearest',cmap ='bwr', vmin = -2, vmax = 2)\n",
    "    counter = counter + 1\n",
    "    plt.yticks(*zip(*list(enumerate(pretty_chars))[::6]))\n",
    "    plt.xlabel(\"dimension\")\n",
    "\n",
    "# import matplotlib as mpl\n",
    "# plt.subplot2grid((3,7), (1,6))\n",
    "# #mpl.colorbar.ColorbarBase(ax)\n",
    "# plt.imshow(np.zeros((0,0)), vmin=-2, vmax=2)\n",
    "# _= fig.colorbar(ax, ticks=[])\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "nVrtOrj4XT6s"
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(10,20))\n",
    " \n",
    "w_inn = np.transpose(b_ixn)\n",
    "w_out_temp = np.transpose(w_out)\n",
    "print w_out_temp.shape\n",
    "print w_inn.shape\n",
    "\n",
    "u,s,v = np.linalg.svd(w_out_temp)\n",
    "temp = w_out_temp.dot(np.transpose(v))\n",
    "b_1_inv = np.transpose(v)\n",
    "b_1 = np.linalg.inv(b_1_inv)\n",
    "q_1 = w_out_temp.dot(b_1_inv)[:27,:27]\n",
    "\n",
    "block_1 = np.identity(num_nodes)\n",
    "for i in range(27):\n",
    "  q_1[i] = q_1[i] / np.linalg.norm(w_out_temp[i])\n",
    "block_1[:27,:27] = q_1\n",
    "\n",
    "\n",
    "\n",
    "plt.subplot(6,1,1)\n",
    "plt.imshow(w_out_temp.dot(b_1_inv))\n",
    "\n",
    "plt.subplot(6,1,2)\n",
    "basis_1 = block_1.dot(b_1)\n",
    "plt.imshow(w_out_temp.dot(np.linalg.inv(basis_1)))\n",
    "\n",
    "plt.subplot(6,1,3)\n",
    "w_til_in = basis_1.dot(w_inn)\n",
    "w_til_in_sub = w_til_in[27:,:]\n",
    "u,s,v = np.linalg.svd(w_til_in_sub)\n",
    "\n",
    "b_2 = np.transpose(u)\n",
    "plt.imshow(np.transpose(b_2.dot(w_til_in_sub)))\n",
    "\n",
    "block_b2 = np.identity(num_nodes)\n",
    "block_b2[27:,27:]=b_2\n",
    "q_2 = b_2.dot(w_til_in_sub)[:27,:27]\n",
    "\n",
    "# for i in range(27):\n",
    "#   q_2[i] = q_2[i] / (np.linalg.norm(w_inn[:,i])\n",
    "\n",
    "block_2 = np.identity(num_nodes)\n",
    "block_2[27:2*27,27:2*27] = q_2\n",
    "\n",
    "plt.subplot(6,1,4)\n",
    "plt.imshow(np.transpose(np.linalg.inv(block_2).dot(block_b2).dot(w_til_in)))\n",
    "\n",
    "#basis_complete maps h to h_tilde, basis_complete.dot(h) = h_tilde\n",
    "basis_complete = np.linalg.inv(block_2).dot(block_b2).dot(basis_1)\n",
    "\n",
    "plt.subplot(6,1,5)\n",
    "plt.imshow(np.transpose(basis_complete.dot(w_inn)),'bwr')\n",
    "\n",
    "\n",
    "plt.subplot(6,1,6)\n",
    "plt.imshow(w_out_temp.dot(np.linalg.inv(basis_complete)),'bwr')\n",
    "\n",
    "random_states_pca_transformed = np.zeros((len(random_states), num_nodes))\n",
    "for i in range(len(random_states)):\n",
    "  random_states_pca_transformed[i] = (basis_complete).dot(np.transpose(random_states[i]))\n",
    "\n",
    "pca_2 = PCA(n_components=pca_dim - 54)\n",
    "pca_2.fit(random_states_pca_transformed[:][54:])\n",
    "#pca_states_transform = random_states_pca[:,54:]\n",
    "#pca_states_transform.var(1).mean()\n",
    "#pca_states_transformed = (basis_complete).dot(np.transpose(random_states_pca))\n",
    "block_3 = np.identity(num_nodes)\n",
    "block_3[54:,54:] = (pca_2.components_)\n",
    "basis_complete = block_3.dot(np.linalg.inv(block_2)).dot(block_b2).dot(basis_1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "OnuvTvSVl1sZ"
   },
   "outputs": [],
   "source": [
    "plt.imshow(w_out.T.dot(b_1.T))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "v_t8WKtjb-9W"
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(8, 8))\n",
    "ax = scatter_plot(df_states[\"state\"], df_states[\"input\"])\n",
    "cbar = fig.colorbar(ax)\n",
    "\n",
    "plt.savefig(\"/tmp/pca_plot_hidden_state_by_input.pdf\") \n",
    "  \n",
    "  #PCA PLOT\n",
    "\n",
    "\n",
    "# np.dot(pca.components_, np.transpose(pca.components_))\n",
    "\n",
    "\n",
    "# w_h_inn = sess.run(source_rnn_cell._Wh_ixnn)\n",
    "# b_ixn = sess.run(source_rnn_cell._b_ixn)\n",
    "# out_inn = sess.run( source_rnn_cell._Wro_nxo)\n",
    "# init =  sess.run( source_rnn_cell._init_vector)\n",
    "\n",
    "# w_h_pca = np.zeros((27, pca_dim**2))\n",
    "# b_h_pca = np.zeros((27, pca_dim))\n",
    "# out_pca = np.dot( (pca.components_), out_inn)\n",
    "# init_pca = np.dot( (pca.components_), np.transpose(init))\n",
    "# w_h_pca_nxn = np.zeros((27, pca_dim, pca_dim ))\n",
    "# w_h_nxn =  np.zeros((27, num_nodes, num_nodes ))\n",
    "# for i in range(27):\n",
    "#   tmp = np.reshape(w_h_inn[i, :], [num_nodes, num_nodes] )\n",
    "#   w_h_nxn[i] = tmp\n",
    "#   tmp_2 = np.dot(pca.components_, np.dot(tmp, np.transpose(pca.components_)))\n",
    "#   w_h_pca[i,:] = np.reshape( tmp_2, [-1]) \n",
    "#   w_h_pca_nxn[i] = tmp_2\n",
    "#   b_h_pca[i,:] =  np.dot(b_ixn[i,:], np.transpose(pca.components_))\n",
    "\n",
    "# source_rnn_cell_pca = source_cell_class(input_size, pca_dim, output_size, hparams, layer = 'only')\n",
    "\n",
    "# _ = sess.run(source_rnn_cell_pca._Wh_ixnn.assign(w_h_pca))\n",
    "# _ = sess.run(source_rnn_cell_pca._b_ixn.assign(b_h_pca))\n",
    "# _ = sess.run( source_rnn_cell_pca._Wro_nxo.assign(out_pca))\n",
    "# out_original_bias = sess.run( source_rnn_cell._bro_o )\n",
    "# _ = sess.run( source_rnn_cell_pca._bro_o.assign(out_original_bias))\n",
    "\n",
    "# _ =  sess.run( source_rnn_cell_pca._init_vector.assign(np.transpose(init_pca)))\n",
    "\n",
    "\n",
    "# _ = sess.run(source_rnn_cell_pca._x0_ixn.assign(np.zeros((10,27,pca_dim))))\n",
    "\n",
    "\n",
    "# state_placeholder_pca = tf.placeholder(tf.float32, shape=[None, pca_dim])\n",
    "# output_source_pca, hidden_state_source_pca = source_rnn_cell_pca(input_placeholder, state_placeholder_pca)\n",
    "\n",
    "\n",
    "\n",
    "# output_eval_pca = tf.nn.softmax( output_source_pca * temperature)\n",
    "\n",
    "# output_placeholder = tf.placeholder(tf.float32, shape=[None, 27])\n",
    "\n",
    "# softmax = tf.nn.softmax( output_placeholder * temperature)\n",
    "# output_eval_pca = tf.nn.softmax( output_source_pca * temperature)\n",
    "\n",
    "# train_loss_np  = tf.reduce_mean(\n",
    "#       tf.nn.softmax_cross_entropy_with_logits(\n",
    "#         output_placeholder, target_placeholder) / tf.log(2.0) )\n",
    "\n",
    "# train_loss_pca  = tf.reduce_mean(\n",
    "#       tf.nn.softmax_cross_entropy_with_logits(\n",
    "#         output_source_pca, target_placeholder) / tf.log(2.0) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "neDn8f5_zFFx"
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(20, 15))\n",
    "losses = []\n",
    "count = 0\n",
    "h  = h_zero\n",
    "counter = 0\n",
    "h_space = h\n",
    "h_other = h\n",
    "input_vals = \" sample\"\n",
    "for i in range(len(input_vals)):\n",
    "  h_in = h\n",
    "  input_val = mapping[input_vals[i]]\n",
    "\n",
    "  h_space, outpt_np_space, loss_space = make_step(h_space, input_val, y[i])\n",
    "  h_other, outpt_np_other, loss_other = make_step(h_other, input_val, y[i])\n",
    "  h, outpt_np, loss =  make_step(h, input_val, y[i])\n",
    "  input_vals += chars[input_val]\n",
    "\n",
    "print(input_vals)\n",
    "backwards_val = \"\"\n",
    "for i in range(len(input_vals)):\n",
    "  probs = np.zeros(27)\n",
    "  for k in range(27):\n",
    "    _, p =make_step_back(h,k)\n",
    "    probs[k] = p\n",
    "  print(probs)\n",
    "  input_val = np.argmax(probs)\n",
    "  h, _ = make_step_back(h,input_val)\n",
    "  backwards_val += chars[input_val]\n",
    "print(backwards_val)\n",
    "  \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "xXIi7leJU0KQ"
   },
   "outputs": [],
   "source": [
    "pca.explained_variance_ratio_[-5:]\n",
    "cond_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "B1T3JJnXASXF"
   },
   "outputs": [],
   "source": [
    "max_history = 5\n",
    "n_random = 20000\n",
    "\n",
    "#define the inputs\n",
    "index = range(n_random) \n",
    "np.random.shuffle(index)\n",
    "inputs = np.zeros((27*num_each_char,27))\n",
    "for i in range(27*num_each_char):\n",
    "  inputs[i, i % 27] = 1\n",
    "\n",
    "# This produces n_random hidden states by running the RNN in feed forward mode with the input. We could add a random sampling probability here.\n",
    "df_states = pd.DataFrame()\n",
    "errors = np.zeros(50)\n",
    "#h  = np.zeros((1, num_nodes))\n",
    "h  = sess.run(source_rnn_cell.zero_state(1, dtype=tf.float32))\n",
    "losses = []\n",
    "\n",
    "history = [0 for i in xrange(max_history)]\n",
    "for i in range(n_random):\n",
    "  if i % 1000 == 0:\n",
    "    print i, len(df_states)\n",
    "  if i % 50 == 0:\n",
    "    h  = sess.run(source_rnn_cell.zero_state(1, dtype=tf.float32))\n",
    "  h_in = h\n",
    "  h,loss = sess.run( [hidden_state_source, train_loss], feed_dict={input_placeholder:x[i], state_placeholder:h, target_placeholder:y[i]})\n",
    "  history.pop(0)\n",
    "  history.append(np.argmax(x[i]))\n",
    "\n",
    "  if i % 50 > 10:\n",
    "    losses.append(loss)\n",
    "    d = {\"index\" : i, \"state\": h, \"input\":np.argmax(x[i]), \"state_in\":h_in, \"delta\":h-h_in, \"dist\": i % 50, \"loss\": loss, \"history\":history[:] }\n",
    "    df_states = df_states.append(d, ignore_index=True)\n",
    "print(\"produced {} random states\".format(n_random))\n",
    "print(np.mean(losses))\n",
    "print len(df_states)\n",
    "\n",
    "random_states = np.array([v[0] for v in df_states[\"state\"].values])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "g_WRo-TOyOBd"
   },
   "outputs": [],
   "source": [
    "#history from nearest neighbor\n",
    "idx = 580\n",
    "s = df_states[\"state\"].values[idx]\n",
    "def get_dist(s1, s2):\n",
    "  d = (s1 - s2)**2\n",
    "  return np.sum(d)\n",
    "\n",
    "dists = [get_dist(s, s1) for s1 in df_states[\"state\"]]\n",
    "sorted_args = np.argsort(dists)\n",
    "print dists[sorted_args[0]], dists[sorted_args[1]]\n",
    "print df_states[\"history\"].values[idx]\n",
    "\n",
    "for i in xrange(10):\n",
    "  print dists[sorted_args[i]], get_chars(df_states[\"history\"].values[sorted_args[i]])\n",
    "  \n",
    "#Can we plot this nicely? Distance in space as a function of difference in characters? Also, I think we should be thinking about angular distance. Eg. cosine distance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "9c-ZcpB899Jj"
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(15, 7))\n",
    "\n",
    "mapping = {}\n",
    "for i in range(27):\n",
    "  mapping[chars[i]] = i\n",
    "  \n",
    "\n",
    "freq = np.zeros(27)\n",
    "for i in x:\n",
    "  freq[np.argmax(i)] +=1\n",
    "  \n",
    "freq = freq / np.sum(freq)\n",
    "plt.plot(freq,'r')\n",
    "                           \n",
    "\n",
    "outpt_np = np.dot( b_h_pca[:,:],out_pca) + out_original_bias\n",
    "\n",
    "temp = b_h_pca\n",
    "temp[:,30:] = 0\n",
    "outpt_corrupt = np.dot( temp,out_pca) + out_original_bias\n",
    "\n",
    "plt.subplot(1,2,1)\n",
    "plt.title(\"Decoded prio vs actual\")\n",
    "val_x =soft_max(np.reshape(out_original_bias, [1,-1]))\n",
    "print val_x.shape\n",
    "print(np.sum(val_x, 1))\n",
    "_ = plt.plot(val_x[0])\n",
    "\n",
    "plt.plot(freq,'r')\n",
    "plt.subplot(1,2,2)\n",
    "plt.title(\"Decoded probabilities from offset\")\n",
    "\n",
    "for i in range(27):\n",
    "  _ =plt.plot(soft_max(outpt_np[i]))\n",
    "  print 'char ', chars[i], 'next ', chars[np.argmax(outpt_np[i])], 'corrupted ', chars[np.argmax(outpt_corrupt[i])]\n",
    "\n",
    "plt.savefig(\"/tmp/prior_vs_actual.pdf\")\n",
    " #Case study: \"United States\" vs \"United Nations\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "0ui3ATkrzogT"
   },
   "outputs": [],
   "source": [
    "#random_states_pca.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "eZDaPUyKneDx"
   },
   "outputs": [],
   "source": [
    "#Sample PCA states and compute PCA error\n",
    "max_history = 5\n",
    "n_random = 10000\n",
    "\n",
    "#define the inputs\n",
    "index = range(n_random) \n",
    "np.random.shuffle(index)\n",
    "inputs = np.zeros((27*num_each_char,27))\n",
    "for i in range(27*num_each_char):\n",
    "  inputs[i, i % 27] = 1\n",
    "\n",
    "#Hack: Overwrite all of the offsets with the same value (since they are correlated)\n",
    "if False:\n",
    "  temp = b_h_pca\n",
    "  for i in range(27):\n",
    "    temp[i] = np.mean(b_h_pca,0) \n",
    "  _ = sess.run(source_rnn_cell_pca._b_ixn.assign(temp))\n",
    "else:\n",
    "  _ = sess.run(source_rnn_cell_pca._b_ixn.assign(b_h_pca))\n",
    "  \n",
    "  \n",
    "# This produces n_random hidden states by running the RNN in feed forward mode with the input. We could add a random sampling probability here.\n",
    "df_states_pca = pd.DataFrame()\n",
    "errors = np.zeros(50)\n",
    "#h  = np.zeros((1, num_nodes))\n",
    "h  = sess.run(source_rnn_cell_pca.zero_state(1, dtype=tf.float32))\n",
    "losses = []\n",
    "counter = 0\n",
    "for i in range(0,n_random):\n",
    "  if i % 1000 == 0:\n",
    "    print i, len(df_states)\n",
    "  if i % 50 == 0:\n",
    "    h  = sess.run(source_rnn_cell_pca.zero_state(1, dtype=tf.float32))\n",
    "  h_in = h\n",
    "  h,loss = sess.run( [hidden_state_source_pca, train_loss_pca], feed_dict={input_placeholder:x[i], state_placeholder_pca:h, target_placeholder:y[i]})\n",
    "\n",
    "  if i % 50 > 10:\n",
    "    losses.append(loss)\n",
    "    d = {\"index\" : i, \"state\": h, \"input\":np.argmax(x[i]), \"state_in\":h_in, \"delta\":h-h_in, \"dist\": i % 50, \"loss\": loss, \"history\":history[:] }\n",
    "    df_states_pca = df_states_pca.append(d, ignore_index=True)\n",
    "print(\"produced {} random states\".format(n_random))\n",
    "print(np.mean(losses))\n",
    "print len(df_states_pca)\n",
    "random_states_pca = np.array([v[0] for v in df_states_pca[\"state\"].values])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "ZptdDIQ1mTcT"
   },
   "outputs": [],
   "source": [
    "vals = np.zeros((len(df_limit_history),max_history))\n",
    "logit_val = np.zeros((len(df_limit_history),max_history))\n",
    "length = np.zeros((len(df_limit_history),max_history))\n",
    "logit_length = np.zeros((len(df_limit_history),max_history))\n",
    "for i in range(len(df_limit_history)):\n",
    "  i_vec = np.zeros((num_nodes,1))\n",
    "  for j in range( min(max_history, i)-1, 0, -1):\n",
    "    vals[i,j] = np.linalg.norm(df_limit_history[\"history\"][i][j])\n",
    "    i_vec = i_vec + df_limit_history[\"history\"][i][j]\n",
    "    length[i,j] = np.linalg.norm(i_vec)\n",
    "    logit_length[i,j] = np.linalg.norm(np.dot( np.reshape(i_vec, [1,-1]), out_pca) )\n",
    "    logit_val[i,j] = np.linalg.norm( np.dot(np.reshape(df_limit_history[\"history\"][i][j], [1,-1]), out_pca))\n",
    "\n",
    "\n",
    "#plt.plot(length.mean(0))\n",
    "plt.subplot(2,2,1)\n",
    "plt.semilogy((vals.mean(0)))\n",
    "plt.subplot(2,2,2)\n",
    "plt.semilogy((length.mean(0)))\n",
    "\n",
    "#plt.plot(length.mean(0))\n",
    "plt.subplot(2,2,3)\n",
    "plt.semilogy((logit_val.mean(0)))\n",
    "plt.subplot(2,2,4)\n",
    "plt.semilogy((logit_length.mean(0)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "Yb1ZVFJHPgsr"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "wRwvUf0WHPwm"
   },
   "outputs": [],
   "source": [
    "vals = np.zeros((len(df_limit_history),max_history))\n",
    "length = np.zeros((len(df_limit_history),max_history))\n",
    "for i in range(len(df_limit_history)):\n",
    "  i_vec = np.zeros((num_nodes,1))\n",
    "  for j in range(max_history-1,0,-1):\n",
    "    vals[i,j] = np.linalg.norm(df_limit_history[\"history\"][i][j])\n",
    "    i_vec = i_vec + df_limit_history[\"history\"][i][j]\n",
    "    length[i,j] = np.linalg.norm(i_vec)\n",
    "\n",
    "len( df_limit_history[\"history\"])\n",
    "#plt.plot(length.mean(0))\n",
    "plt.subplot(1,2,1)\n",
    "plt.loglog((vals.mean(0)))\n",
    "plt.subplot(1,2,2)\n",
    "plt.semilogy((vals.mean(0)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "BNnA4jYSO-9R"
   },
   "outputs": [],
   "source": [
    "#Fig7\n",
    "plt.subplot(1,2,1)\n",
    "plt.semilogy((length.mean(0))[:-1])\n",
    "plt.semilogy((vals.mean(0))[:-1])\n",
    "plt.subplot(1,2,2)\n",
    "\n",
    "plt.loglog((length.mean(0))[:-1])\n",
    "plt.loglog((vals.mean(0))[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "YqwTSqry8WSg"
   },
   "outputs": [],
   "source": [
    "#q, r = np.linalg.qr(out_pca)\n",
    "# h  = sess.run(source_rnn_cell_pca.zero_state(1, dtype=tf.float32))\n",
    "# zero_state = h.dot(q)\n",
    "# losses = []\n",
    "# counter = 0\n",
    "# temp = np.copy(b_h_pca)\n",
    "# temp_offset = np.dot(b_h_pca,q)\n",
    "# temp_matix = np.zeros((27,27,27))\n",
    "\n",
    "# for i in range(27):\n",
    "#   temp_matix[i] = np.dot(np.transpose(q), np.dot(w_h_pca_nxn[i],q))\n",
    "\n",
    "test = np.zeros((num_nodes, num_nodes))\n",
    "test[:,:27] = out_pca \n",
    "\n",
    "u,s,v = np.linalg.svd(test)\n",
    "u[:,27].dot(out_pca)\n",
    "basis = np.transpose(u)\n",
    "basis_inv = np.linalg.inv(basis)\n",
    "\n",
    "# out_pca = np.dot( (pca.components_), out_inn)\n",
    "# init_pca = np.dot( (pca.components_), np.transpose(init))\n",
    "# w_h_pca_nxn = np.zeros((27, pca_dim, pca_dim ))\n",
    "# w_h_nxn =  np.zeros((27, num_nodes, num_nodes ))\n",
    "# for i in range(27):\n",
    "#   tmp = np.reshape(w_h_inn[i, :], [num_nodes, num_nodes] )\n",
    "#   w_h_nxn[i] = tmp\n",
    "#   tmp_2 = np.dot(pca.components_, np.dot(tmp, np.transpose(pca.components_)))\n",
    "#   w_h_pca[i,:] = np.reshape( tmp_2, [-1]) \n",
    "#   w_h_pca_nxn[i] = tmp_2\n",
    "\n",
    "\n",
    "#Barebones running of the RNN in reduced hidden state space of 27 x 27.\n",
    "#h  = np.zeros((1, num_nodes))\n",
    "h  = sess.run(source_rnn_cell_pca.zero_state(1, dtype=tf.float32))\n",
    "losses = []\n",
    "counter = 0\n",
    "temp = np.copy(b_h_pca)\n",
    "temp_offset = np.zeros((27, pca_dim)) \n",
    "temp_matix = np.zeros((27,pca_dim,pca_dim))\n",
    "temp_out_pca = basis.dot(out_pca)\n",
    "zero_state = basis.dot(np.transpose(h))\n",
    "\n",
    "for i in range(27):\n",
    "  temp_matix[i] = np.dot(basis, np.dot(w_h_pca_nxn[i],basis_inv))\n",
    "  temp_offset[i] = np.dot(b_h_pca[i,:], np.transpose(basis)) #     np.transpose(np.dot(basis_inv, np.transpose(b_h_pca))) \n",
    "    \n",
    "\n",
    "for i in range(10000,12000):\n",
    "  if i % 50 == 0:\n",
    "    h  = zero_state\n",
    "  h_in = h\n",
    "  input_val = np.argmax(x[i])\n",
    "  h = np.dot( temp_matix[input_val], np.reshape( h, [-1,1] )) + np.reshape(temp_offset[input_val], [-1,1] ) \n",
    "  \n",
    "  outpt_np = np.dot( np.reshape(h, [1,-1]),temp_out_pca) + out_original_bias\n",
    "  loss = sess.run( [train_loss_np], feed_dict={output_placeholder: outpt_np, target_placeholder:y[i]})\n",
    "  count += 1\n",
    "  if i % 50 > 10:\n",
    "    losses.append(loss)\n",
    "print(\"produced {} random states\".format(n_random))\n",
    "print(np.mean(losses))\n",
    "print len(df_states_pca)\n",
    "\n",
    "#print(np.dot(np.transpose(q),q ))\n",
    "\n",
    "#print(abs((out_inn - q.dot(r)))).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "nb9w3SRQ5qLY"
   },
   "outputs": [],
   "source": [
    "#Barebones running of the RNN with all elements accessible in numpy.\n",
    "#h  = np.zeros((1, num_nodes))\n",
    "h  = sess.run(source_rnn_cell_pca.zero_state(1, dtype=tf.float32))\n",
    "losses = []\n",
    "counter = 0\n",
    "temp = np.copy(b_h_pca)\n",
    "if True:\n",
    "  #temp[:,:] = 0\n",
    "  offset_for_propagation =temp  \n",
    "  offset_for_readout = b_h_pca \n",
    "\n",
    "\n",
    "for i in range(10000,12000):\n",
    "  if i % 50 == 0:\n",
    "    h  = sess.run(source_rnn_cell_pca.zero_state(1, dtype=tf.float32))\n",
    "  h_in = h\n",
    "  input_val = np.argmax(x[i])\n",
    "  h = np.dot( w_h_pca_nxn[input_val], np.reshape( h, [-1,1] )) + np.reshape(offset_for_readout[input_val], [-1,1] ) \n",
    "  \n",
    "  outpt_np = np.dot( np.reshape(h, [1,-1]), out_pca) + out_original_bias\n",
    "  loss = sess.run( [train_loss_np], feed_dict={output_placeholder: outpt_np, target_placeholder:y[i]})\n",
    "  h = h -  np.reshape(offset_for_readout[input_val], [-1,1] ) + np.reshape(offset_for_propagation[input_val], [-1,1] )\n",
    "  count += 1\n",
    "  if i % 50 > 10:\n",
    "    losses.append(loss)\n",
    "print(\"produced {} random states\".format(n_random))\n",
    "print(np.mean(losses))\n",
    "print len(df_states_pca)\n",
    "\n",
    "#can take out the offset for current step and get reduction to 2.0\n",
    "#if taken out from the propagation, error goes up to 3.75"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "seNaZ-xrYgSm"
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(10,20))\n",
    "\n",
    "w_out = np.transpose(out_pca)\n",
    "w_inn = np.transpose(b_h_pca)\n",
    "print w_out.shape\n",
    "print w_inn.shape\n",
    "\n",
    "u,s,v = np.linalg.svd(w_out)\n",
    "temp = w_out.dot(np.transpose(v))\n",
    "b_1_inv = np.transpose(v)\n",
    "b_1 = np.linalg.inv(b_1_inv)\n",
    "q_1 = w_out.dot(b_1_inv)[:27,:27]\n",
    "\n",
    "block_1 = np.identity(num_nodes)\n",
    "for i in range(27):\n",
    "  q_1[i] = q_1[i] / np.linalg.norm(w_out[i])\n",
    "block_1[:27,:27] = q_1\n",
    "\n",
    "\n",
    "\n",
    "plt.subplot(6,1,1)\n",
    "plt.imshow(w_out.dot(b_1_inv))\n",
    "\n",
    "plt.subplot(6,1,2)\n",
    "basis_1 = block_1.dot(b_1)\n",
    "plt.imshow(w_out.dot(np.linalg.inv(basis_1)))\n",
    "\n",
    "plt.subplot(6,1,3)\n",
    "w_til_in = basis_1.dot(w_inn)\n",
    "w_til_in_sub = w_til_in[27:,:]\n",
    "u,s,v = np.linalg.svd(w_til_in_sub)\n",
    "\n",
    "b_2 = np.transpose(u)\n",
    "plt.imshow(np.transpose(b_2.dot(w_til_in_sub)))\n",
    "\n",
    "block_b2 = np.identity(num_nodes)\n",
    "block_b2[27:,27:]=b_2\n",
    "q_2 = b_2.dot(w_til_in_sub)[:27,:27]\n",
    "\n",
    "# for i in range(27):\n",
    "#   q_2[i] = q_2[i] / (np.linalg.norm(w_inn[:,i])\n",
    "\n",
    "block_2 = np.identity(num_nodes)\n",
    "block_2[27:2*27,27:2*27] = q_2\n",
    "\n",
    "plt.subplot(6,1,4)\n",
    "plt.imshow(np.transpose(np.linalg.inv(block_2).dot(block_b2).dot(w_til_in)))\n",
    "\n",
    "#basis_complete maps h to h_tilde, basis_complete.dot(h) = h_tilde\n",
    "basis_complete = np.linalg.inv(block_2).dot(block_b2).dot(basis_1)\n",
    "\n",
    "plt.subplot(6,1,5)\n",
    "plt.imshow(np.transpose(basis_complete.dot(w_inn)),'bwr')\n",
    "\n",
    "\n",
    "plt.subplot(6,1,6)\n",
    "plt.imshow(w_out.dot(np.linalg.inv(basis_complete)),'bwr')\n",
    "\n",
    "random_states_pca_transformed = np.zeros(random_states_pca.shape)\n",
    "for i in range(len(random_states_pca)):\n",
    "  random_states_pca_transformed[i] = (basis_complete).dot(np.transpose(random_states_pca[i]))\n",
    "\n",
    "pca_2 = PCA(n_components=pca_dim - 54)\n",
    "pca_2.fit(random_states_pca_transformed[:,54:])\n",
    "#pca_states_transform = random_states_pca[:,54:]\n",
    "#pca_states_transform.var(1).mean()\n",
    "#pca_states_transformed = (basis_complete).dot(np.transpose(random_states_pca))\n",
    "block_3 = np.identity(num_nodes)\n",
    "block_3[54:,54:] = (pca_2.components_)\n",
    "basis_complete = block_3.dot(np.linalg.inv(block_2)).dot(block_b2).dot(basis_1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "WuKa6i_yDws4"
   },
   "outputs": [],
   "source": [
    "random_states_pca_transformed = np.zeros(random_states_pca.shape)\n",
    "for i in range(len(random_states_pca)):\n",
    "  random_states_pca_transformed[i] = (basis_complete).dot(np.transpose(random_states_pca[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "Eru9K7DnkYq7"
   },
   "outputs": [],
   "source": [
    "var = abs(random_states_pca_transformed).max(0)\n",
    "basis_complete_2 = np.copy(basis_complete)\n",
    "\n",
    "for i in range(pca_dim):\n",
    "  basis_complete_2[i] = basis_complete[i] / var[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "S-1KcvdJkSfz"
   },
   "outputs": [],
   "source": [
    "corr = np.corrcoef(np.transpose(basis_complete_2))\n",
    "plt.imshow(corr, 'bwr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "z3olEUVcIxaU"
   },
   "outputs": [],
   "source": [
    "print out_pca.shape\n",
    "\n",
    "u,s,v = np.linalg.svd(out_pca)\n",
    "\n",
    "outpt_np = np.dot( np.reshape(h, [1,-1]), out_pca)\n",
    "u[:,27].dot(out_pca)\n",
    "basis_2 = np.transpose(u)\n",
    "\n",
    "print u.shape, s.shape, v.shape, out_pca.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "4wYFISTK72Oh"
   },
   "outputs": [],
   "source": [
    "temp.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "2MZoT77wLIlI"
   },
   "outputs": [],
   "source": [
    "temp[50:55, 0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "mZM4Mi1L8Yfu"
   },
   "outputs": [],
   "source": [
    "#tytyt2\n",
    "# for i in range(29)\"\n",
    "fig = plt.figure(figsize=(30, 30))\n",
    "for i in range(27):\n",
    "  plt.subplot(14,2,i+1)\n",
    "  plt.title(chars[i])\n",
    "  \n",
    "\n",
    "  #_ = plt.imshow(np.transpose( w_h_pca_nxn[i].dot(out_pca)), cmap ='bwr',vmin=-4,vmax=4)\n",
    "  #_ = plt.imshow(np.transpose( w_h_nxn[i].dot(out_inn)), cmap ='bwr')#,vmin=-4,vmax=4)\n",
    "  _ = plt.imshow(np.transpose( temp_matix[i].dot(temp_out_pca)), cmap ='bwr')#,vmin=-4,vmax=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "U_3s9MMffeB2"
   },
   "outputs": [],
   "source": [
    "all_state, _, next_val, input_val, suprise, input_indx, certainty = predict(\" testing\", 5, 12.0) #basis\n",
    "print next_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "-fSDXq3j-kfy"
   },
   "outputs": [],
   "source": [
    "plt.subplot?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "CYRaIi3K99QI"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "pjAePAiqyRus"
   },
   "outputs": [],
   "source": [
    "\n",
    "#Fig_COPY_1 / mast_COPY_plot\n",
    "string = \" approximation for this \"\n",
    "errors = np.zeros(50)\n",
    "\n",
    "fig = plt.figure(figsize=(20, 8))\n",
    "\n",
    "#h  = np.zeros((1, num_nodes))\n",
    "h  = sess.run(source_rnn_cell_pca.zero_state(1, dtype=tf.float32))\n",
    "history = []\n",
    "history_cont = []\n",
    "input_string = string\n",
    "max_history = len(input_string)\n",
    "out_history = np.ones((max_history, 27))\n",
    "\n",
    "canvas = np.zeros((len(string)*27, 2*max_history + 4))\n",
    "probabilities = np.zeros((len(string)*27,1))\n",
    "\n",
    "# df_limit_history = pd.DataFrame()\n",
    "#h  = sess.run(source_rnn_cell_pca.zero_state(1, dtype=tf.float32))\n",
    "#history.append(h)\n",
    "count = 0\n",
    "for i in range(len(input_string)):\n",
    "  h_in = h\n",
    "  input_val = mapping[input_string[i]]\n",
    "  sum_of_hm1_cont = np.zeros((num_nodes,1))\n",
    "  for k in range(len(history)):\n",
    "    new_h = np.dot( w_h_pca_nxn[input_val], np.reshape( history[k], [-1,1] ))\n",
    "    history[k] = new_h\n",
    "    sum_of_hm1_cont = sum_of_hm1_cont + new_h\n",
    "\n",
    "  new_offset = np.reshape(b_h_pca[input_val], [-1,1])\n",
    "  history.append(new_offset)\n",
    "  h = sum_of_hm1_cont +  new_offset\n",
    "  history_cont = []\n",
    "  for k in range(len(history)):\n",
    "    canvas[i*27 : (i+1) * 27,(k+1)*2:(k+2)*2] = np.reshape( np.dot( np.reshape(history[k], [1,-1]), out_pca), [-1,1])\n",
    "    history_cont.append( np.dot( np.reshape(history[k], [1,-1]), out_pca) )\n",
    "  outpt_np = np.dot( np.reshape(h, [1,-1]), out_pca) + out_original_bias\n",
    "  probabilities[i*27 : (i+1) * 27] = np.reshape( soft_max(outpt_np) ,[-1,1])\n",
    "  \n",
    "  canvas[i*27 : (i+1) * 27,-2:]  =  np.reshape(outpt_np, [-1,1])\n",
    "  canvas[i*27 : (i+1) * 27,-4:-2]  = np.reshape(out_original_bias,[-1,1])\n",
    "  \n",
    "#   d = {\"index\" : i, \"state\": h, \"input\":input_val, \"state_in\":h_in, \"delta\":h-h_in, \"loss\": loss, \"history\":history[:], \"output\": outpt_np, \"props\": soft_max(outpt_np), \"history_cont\": history_cont[:], \"char\":input_string[i] }\n",
    "#   df_limit_history = df_limit_history.append(d, ignore_index=True)\n",
    "\n",
    "\n",
    "fig = plt.figure(figsize=(20, 5))\n",
    "\n",
    "\n",
    "\n",
    "plt.subplot(2,1,1)    \n",
    "ax  = plt.imshow(np.transpose(canvas), 'jet', interpolation = 'nearest', vmin = -5, vmax = 10)\n",
    "_=plt.xticks( np.arange(0,max_history*27,27) )\n",
    "\n",
    "\n",
    "#fig.colorbar(ax)\n",
    "\n",
    "plt.subplot(2,1,2)    \n",
    "_ = plt.plot( probabilities) \n",
    "_ = plt.xticks( np.arange(0,max_history*27,27) )\n",
    "\n",
    "for i in range(max_history):\n",
    "  val =   np.argmax(probabilities[i*27 : (i+1) * 27])\n",
    "  plt.text(i*27 + val, probabilities[i*27 + val] , chars[val])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "65hYY_fW0go0"
   },
   "outputs": [],
   "source": [
    "hist = df_limit_history[\"history_cont\"][3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "MZ0K8Ua600q4"
   },
   "outputs": [],
   "source": [
    "hist[0].shape, len(hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "2IcwGOn-zd1g"
   },
   "outputs": [],
   "source": [
    "i = 5\n",
    "print canvas.shape, len(string)\n",
    "canvas[i*27:(i+1) * 27,:].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "KLorwa6nOkvZ"
   },
   "outputs": [],
   "source": [
    "?np.random.uniform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "BLAcuhMY03zr"
   },
   "outputs": [],
   "source": [
    "index = 11\n",
    "hist = df_limit_history[\"history_cont\"][index]\n",
    "\n",
    "hist_negative = [np.zeros(27) for i in xrange(len(hist))]\n",
    "hist_positive = [np.zeros(27) for i in xrange(len(hist))]\n",
    "for i in xrange(len(hist)):\n",
    "  for j in xrange(len(hist[i][0])):\n",
    "    if hist[i][0][j] < 0:\n",
    "      hist_negative[i][j] = hist[i][0][j]     \n",
    "    else:\n",
    "      hist_positive[i][j] = hist[i][0][j]\n",
    "\n",
    "ind = np.arange(27)    # the x locations for the groups\n",
    "width = 0.3       # the width of the bars: can also be len(x) sequence\n",
    "\n",
    "summed = np.zeros(27)\n",
    "#positive_summed = np.zeros(27)\n",
    "\n",
    "colors = [np.random.uniform(size =3) for i in xrange(len(hist))]\n",
    "\n",
    "axes = []\n",
    "for i in xrange(len(hist_negative)):\n",
    "  p = plt.bar(ind, hist_negative[i], width,bottom = summed, color = colors[i])\n",
    "  summed+=hist_negative[i]\n",
    "  axes.append(p)\n",
    "\n",
    "for i in xrange(len(hist_positive)):\n",
    "  p2 = plt.bar(ind + width, hist_positive[i], width, bottom = summed, color = colors[i])\n",
    "  summed+=hist_positive[i]\n",
    "\n",
    "plt.ylabel('Scores')\n",
    "plt.title('Scores by group and gender')\n",
    "plt.xticks(ind + width/2.)\n",
    "plt.yticks(np.arange(0, 1, 10))\n",
    "plt.legend(axes, df_limit_history[\"char\"].values)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "SVWDh_C_03qA"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "UEEujyhew1YU"
   },
   "outputs": [],
   "source": [
    "q, r = np.linalg.qr(out_pca)\n",
    "\n",
    "print(abs((out_pca - q.dot(r)))).mean()\n",
    "#print(abs((np.identity - ))).mean()\n",
    "temp_offset = np.dot(b_h_pca,q)\n",
    "#print temp_offset.shape\n",
    "#i = 0\n",
    "#print np.dot(np.transpose(q), np.dot(w_h_pca_nxn[i],q)).shape\n",
    "\n",
    "plt.plot(q.dot(np.transpose(q)).diagonal())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "wVgVCrF50qO5"
   },
   "outputs": [],
   "source": [
    "#this cell builds the bigram_freq and cond_probs dicts. \n",
    "#bigram_freq[(char1, char2)] contains the counts of the bigram char1,char2 (both integers in [0,26]\n",
    "#cond_probs[(char1, char2)] contains prob(char2 | char1)\n",
    "\n",
    "\n",
    "from collections import Counter\n",
    "bigram_freq = Counter()\n",
    "\n",
    "for i in xrange(len(x)-1):\n",
    "  bigram_freq[(np.argmax(x[i]), np.argmax(x[i+1]))] +=1\n",
    "\n",
    "cond_probs = Counter()\n",
    "\n",
    "def sum_cond(char_dict, char):\n",
    "  return np.sum([char_dict[(char, i)] for i in xrange(27)])\n",
    "\n",
    "for char in xrange(27):\n",
    "  total_count = sum_cond(bigram_freq, char)\n",
    "  for char2 in xrange(27):\n",
    "    cond_probs[(char, char2)] = float(bigram_freq[(char, char2)])/total_count\n",
    "  print char, total_count, sum_cond(cond_probs, char)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "vIZCgJIZ25Hj"
   },
   "outputs": [],
   "source": [
    "outpt_np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "cuijRKmB3e17"
   },
   "outputs": [],
   "source": [
    "w_out.shape, w_inn.shape, bias_output.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "0J4Zb4A43MLq"
   },
   "outputs": [],
   "source": [
    "#plot the conditional probs for a given char\n",
    "char = 0\n",
    "bias_output = np.transpose(np.dot(w_out, w_inn)) + out_original_bias\n",
    "plt.plot(soft_max(bias_output[char]), \"r\", label=\"soft_max(W_out * b_x)\")\n",
    "probs = [cond_probs[char, i] for i in xrange(27)]\n",
    "plt.plot(probs, \"g\", label = \"empirical\")\n",
    "plt.xlim(0, 26)\n",
    "plt.xlabel(\"Output Character\")\n",
    "plt.ylabel(\"Conditional Probability\")\n",
    "plt.title(\"P(char | space)\")\n",
    "plt.legend(loc = 0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "iu8npIrCMA46"
   },
   "outputs": [],
   "source": [
    "#compare correlation with bigram probs vs input basis probs\n",
    "fig = plt.figure(figsize=(10, 7))\n",
    "corrs = []\n",
    "corrs_uni = []\n",
    "for char in xrange(27):\n",
    "  probs_network = soft_max(bias_output[char])\n",
    "  cond_p = [cond_probs[char,i] for i in xrange(27)]\n",
    "  corr = np.corrcoef(probs_network, cond_p)[0][1]\n",
    "  corr_unigram = np.corrcoef(cond_p, freq)[0][1]\n",
    "  print corr, corr_unigram\n",
    "  corrs.append(corr)\n",
    "  corrs_uni.append(corr_unigram)\n",
    "  \n",
    "ax = plt.scatter(corrs_uni, corrs, c = range(27), cmap = \"hsv\",  s = 60)\n",
    "plt.xlim(0, 1.0)\n",
    "plt.ylim(0, 1.0)\n",
    "plt.xlabel(\"unigram correlation\")\n",
    "plt.ylabel(\"offset correlation\")\n",
    "plt.title(\"offset vs unigram correlation\")\n",
    "x_pts = np.linspace(0, 1, 100)\n",
    "plt.plot(x_pts, x_pts, \"r\")\n",
    "cbar = plt.colorbar(ax)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "gbYxI4U2Fhyv"
   },
   "outputs": [],
   "source": [
    "print out_pca.shape, basis_5.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "DuLV_aCCjj8m"
   },
   "outputs": [],
   "source": [
    "w_out_trans = w_out.dot(np.linalg.inv(basis_complete))\n",
    "print w_out_trans.shape\n",
    "w_in_trans = basis_complete.dot(w_inn)\n",
    "print w_inn.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "n7zed08nRs-m"
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "#Same plot\n",
    "fig = plt.figure(figsize=(15, 7))\n",
    "\n",
    "mapping = {}\n",
    "for i in range(27):\n",
    "  mapping[chars[i]] = i\n",
    "  \n",
    "\n",
    "freq = np.zeros(27)\n",
    "for i in x:\n",
    "  freq[np.argmax(i)] +=1\n",
    "  \n",
    "freq = freq / np.sum(freq)\n",
    "                            \n",
    "\n",
    "#outpt_np = np.dot( b_h_pca[:,:],out_pca) + out_original_bias\n",
    "\n",
    "#temp = b_h_pca\n",
    "#temp[:,30:] = 0\n",
    "outpt_np = np.dot(b_[:27],temp_out_pca) + out_original_bias\n",
    "#outpt_corrupt = np.dot(temp_offset[:,temp_out_pca) + out_original_bias\n",
    "\n",
    "\n",
    "plt.subplot(1,2,1)\n",
    "plt.title(\"Decoded prio vs actual\")\n",
    "val_x =soft_max(np.reshape(out_original_bias, [1,-1]))\n",
    "print val_x.shape\n",
    "print(np.sum(val_x, 1))\n",
    "_ = plt.plot(val_x[0])\n",
    "\n",
    "plt.plot(freq,'r')\n",
    "plt.subplot(1,2,2)\n",
    "plt.title(\"Decoded probabilities from offset\")\n",
    "\n",
    "for i in range(27):\n",
    "  _ =plt.plot(soft_max(outpt_np[i]))\n",
    "  print 'char ', chars[i], 'next ', chars[np.argmax(outpt_np[i])], 'corrupted ', chars[np.argmax(outpt_corrupt[i])]\n",
    "\n",
    "plt.savefig(\"/tmp/prior_vs_actual.pdf\")\n",
    " #Case study: \"United States\" vs \"United Nations\"\n",
    "\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "R8s5Q6v6TMuG"
   },
   "outputs": [],
   "source": [
    "#compare correlation with bigram probs vs input basis probs\n",
    "corrs = []\n",
    "corrs_uni = []\n",
    "for char in xrange(27):\n",
    "  probs_network = soft_max(outpt_np[char])\n",
    "  cond_p = [cond_probs[char,i] for i in xrange(27)]\n",
    "  corr = np.corrcoef(probs_network, cond_p)[0][1]\n",
    "  corr_unigram = np.corrcoef(cond_p, freq)[0][1]\n",
    "  print corr, corr_unigram\n",
    "  corrs.append(corr)\n",
    "  corrs_uni.append(corr_unigram)\n",
    "  \n",
    "plt.scatter(corrs_uni, corrs, c = range(27), cmap = \"hsv\")\n",
    "plt.xlim(0, 1.0)\n",
    "plt.ylim(0, 1.0)\n",
    "plt.xlabel(\"unigram correlation\")\n",
    "plt.ylabel(\"offset correlation\")\n",
    "plt.title(\"offset vs unigram correlation\")\n",
    "x_pts = np.linspace(0, 1, 100)\n",
    "plt.plot(x_pts, x_pts, \"r\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "XQigV6CvApwc"
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(24, 10))\n",
    "\n",
    "i = mapping['d']\n",
    "plt.subplot(1,2,1)\n",
    "plt.imshow(w_h_pca_nxn[i],'bwr')\n",
    "plt.subplot(1,2,2)\n",
    "w_h_trans = basis_complete_2.dot(w_h_pca_nxn[i]).dot(np.linalg.inv(basis_complete_2))\n",
    "ax = plt.imshow(w_h_trans,'bwr', interpolation = 'nearest')\n",
    "fig.colorbar(ax)\n",
    "#transformed_imag=  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "TgilnHndaCkT"
   },
   "outputs": [],
   "source": [
    "h_in = np.zeros((1, num_nodes))\n",
    "all_state, _, next_val, input_val, suprise, input_indx, certainty = predict(\"g\", 5, 10.0, h= h_in) #basis\n",
    "print next_val\n",
    "\n",
    "all_state, _, next_val, input_val, suprise, input_indx, certainty = predict(\"g\", 5, 10.0) #basis\n",
    "print next_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "iqClr_UBDuRC"
   },
   "outputs": [],
   "source": [
    "#fi5\n",
    "fig = plt.figure(figsize=(15, 10))\n",
    "\n",
    "corrections = w_h_trans[0:27, 0:54]\n",
    "ax = plt.imshow(corrections,'bwr', interpolation = 'none')\n",
    "fig.colorbar(ax)\n",
    "mapping['d']\n",
    "\n",
    "#\"x + E * x + e\n",
    "for i in range(27):\n",
    "  plt.text(27+(i-1.5)*1, 0, chars[i])\n",
    "  plt.text(0, (i-1)*1.05, chars[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "AaiO2oYablPX"
   },
   "outputs": [],
   "source": [
    "all_state, _, next_val, input_val, suprise, input_indx, certainty = predict(\"doctors wear scrubs and\", 500, 1) #basis\n",
    "print next_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "tARHn_mysYnu"
   },
   "outputs": [],
   "source": [
    "#Figure1\n",
    "fig = plt.figure(figsize=(24, 8))\n",
    "\n",
    "\n",
    "\n",
    "plt.subplot(3,2,1)\n",
    "plt.title(\"Readout matrix in original\")\n",
    "\n",
    "plt.imshow(np.transpose(out_inn),interpolation = 'none',cmap ='bwr')\n",
    "\n",
    "plt.subplot(3,2,2)\n",
    "plt.title(\"Offset vectors in original\")\n",
    "plt.imshow((b_ixn), cmap = 'bwr', interpolation = 'none')\n",
    "\n",
    "\n",
    "plt.subplot(3,2,3)\n",
    "plt.title(\"Readout matrix in Pca\")\n",
    "plt.imshow(np.transpose(out_pca),interpolation = 'none',cmap ='bwr')\n",
    "\n",
    "plt.subplot(3,2,4)\n",
    "plt.title(\"Offset vectors in PCA\")\n",
    "\n",
    "plt.imshow((b_h_pca), cmap = 'bwr', interpolation = 'none')\n",
    "\n",
    "\n",
    "plt.subplot(3,2,5)\n",
    "plt.title(\"Readout in Readout basis\")\n",
    "plt.imshow(np.transpose(temp_out_pca), cmap = 'bwr', interpolation = 'none')\n",
    "\n",
    "\n",
    "\n",
    "plt.subplot(3,2,6)\n",
    "plt.title(\"Offset vectors in Readout basis\")\n",
    "\n",
    "#plt.imshow(w_h_pca_nxn[17][:,:], cmap = 'bwr', interpolation = 'none')\n",
    "#plt.imshow(np.transpose(out_pca)>0,interpolation = 'none',cmap ='gray')\n",
    "#plt.plot(np.sum(np.transpose(out_pca)**2,0))\n",
    "\n",
    "plt.imshow((temp_offset), cmap = 'bwr', interpolation = 'none')\n",
    "\n",
    "plt.savefig(\"/tmp/readout_and_offset_in_pca.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "w45xq8in8IIP"
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(5, 5))\n",
    "mat = np.zeros((10,10))\n",
    "mat[:,1]=10\n",
    "mat[:,2:3]=5\n",
    "plt.imshow(mat, cmap = 'bwr', interpolation = 'none')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "RdRL51Z-6wu4"
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(22, 16))\n",
    "\n",
    "for i in range(27):\n",
    "  plt.subplot(5,6, i + 1)\n",
    "  plt.imshow(w_h_pca_nxn[i][20:180,20:180], cmap = 'bwr', interpolation = 'none')\n",
    "  #plt.imshow(np.log(w_h_pca_nxn[i]**2), cmap = 'bwr', interpolation = None)\n",
    "\n",
    "  #plt.imshow(w_h_nxn[i], cmap = 'bwr', interpolation = 'none')\n",
    "  plt.title(chars[i])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "TWavoYdMbStl"
   },
   "outputs": [],
   "source": [
    "#w_h_pca_nxn  --27 weights for transition\n",
    "#out_pca  -- readout matrix \n",
    "#b_h_pca   --27 offset per character \n",
    "#out_original_bias --offset vector for readout. Explained: Predicts the average character probability \n",
    "\n",
    "#   w_h_nxn\n",
    "#   b_ixn\n",
    "#   out_inn\n",
    "#   out_original_bias\n",
    "\n",
    "\n",
    "  \n",
    "\n",
    "plt.subplot(1,2,1)\n",
    "lines = np.zeros((27,num_nodes))\n",
    "for i in range(27):\n",
    "  lines[i] =np.sum(w_h_pca_nxn[i]**2,0)\n",
    "_ =plt.plot(np.mean((np.transpose(lines))**0.5,1))\n",
    "\n",
    "plt.subplot(1,2,2)\n",
    "lines = np.zeros((27,num_nodes))\n",
    "for i in range(27):\n",
    "  lines[i] =np.sum(w_h_pca_nxn[i]**2,1)\n",
    "  \n",
    "_ =plt.plot((((lines))**0.5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "xbtXZRMHu7fz"
   },
   "outputs": [],
   "source": [
    "test = np.zeros((num_nodes, num_nodes))\n",
    "test[:,:27] = out_pca \n",
    "\n",
    "u,s,v = np.linalg.svd(test)\n",
    "\n",
    "u[:,27].dot(out_pca)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "ZeStIVXEygYz"
   },
   "outputs": [],
   "source": [
    "corr = np.corrcoef(np.transpose(w_h_pca_nxn[5]), np.transpose(out_pca))\n",
    "#corr = np.corrcoef((w_h_pca_nxn[0]), np.transpose(out_pca))\n",
    "corr_2 = np.corrcoef(np.transpose(w_h_pca_nxn[26]), np.transpose(out_pca))\n",
    "\n",
    "#tytyt\n",
    "\n",
    "print out_pca.shape\n",
    "w_h_pca_nxn[2].shape\n",
    "plt.subplot(1,2,1)\n",
    "plt.imshow(corr, cmap = 'gray')\n",
    "\n",
    "plt.subplot(1,2,2)\n",
    "\n",
    "_ = plt.hist((np.reshape(corr[num_nodes:, : num_nodes], [-1])), bins = 100)\n",
    "_ = plt.hist(corr_2[num_nodes:, : num_nodes],  bins = 100, hold = True, alpha = 0.5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "i1cohtjO9y4x"
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(16, 16))\n",
    "\n",
    "# for i in range(27):\n",
    "#   plt.subplot(27,1, i + 1)\n",
    "#   plt.imshow(np.transpose(w_h_pca_nxn[i][:,-13:-1]), cmap = 'bwr', interpolation = 'none')\n",
    "#   plt.title(chars[i])\n",
    "  \n",
    "  \n",
    "_ = plt.hist(np.reshape(w_h_pca_nxn,[-1]),  bins = 200, hold = True, alpha = 0.5)\n",
    "plt.yscale('log', nonposy='clip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "rZOncUlMEGeN"
   },
   "outputs": [],
   "source": [
    "w_out.shape, b_ixn.shape, bias_output.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "EZYQbJTOLi4I"
   },
   "outputs": [],
   "source": [
    "def entropy_probs(dist):\n",
    "  return np.sum([-np.log2(p+1e-7)*p for p in dist])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "m80bD2CzNIQG"
   },
   "outputs": [],
   "source": [
    "entropy_probs([.125 for i in xrange(8)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "cG3oEUiAFXWA"
   },
   "outputs": [],
   "source": [
    "char = mapping['a']\n",
    "cond_dist = [cond_probs[(char,i)] for i in xrange(27)]\n",
    "plt.plot(cond_dist)\n",
    "plt.xticks(range(27), chars)\n",
    "print entropy_probs(cond_dist)\n",
    "pass\n",
    "\n",
    "def get_cond_dist(char, use_log = False):\n",
    "  if use_log:\n",
    "    return\n",
    "  return [cond_probs[(char,i)] for i in xrange(27)]\n",
    "def kl_prob(p, q):\n",
    "  return np.sum([p[i] * np.log2(1e-6 + p[i]/(q[i] + 1e-6)) for i in xrange(len(p))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "uHT3IVeMUjST"
   },
   "outputs": [],
   "source": [
    "seaborn.set_style(\"whitegrid\")\n",
    "#offset plot\n",
    "fig = plt.figure(figsize=(20, 6))\n",
    "\n",
    "norms = np.zeros(27)\n",
    "\n",
    "for i in range(27):\n",
    "  norms[i] =( np.linalg.norm(b_ixn[i]))\n",
    "  #print chars[ i ] , np.linalg.norm(b_h_pca[i])\n",
    "\n",
    "plt.subplot(1,3,1)\n",
    "ax = plt.scatter(np.log(freq)/np.log(2),norms, c = np.arange(27), cmap = 'bwr', s = 80)\n",
    "plt.title(\"Norm of offset vector vs entropy in the character\")\n",
    "plt.xlabel(\"bits of entropy\")\n",
    "plt.ylabel(\"norm of offset vector\")\n",
    "\n",
    "plt.subplot(1,3,2)\n",
    "plt.title(\"readout of characters\")\n",
    "\n",
    "#next_probs = np.array([[cond_probs[(i,j)] for j in xrange(27)] for i in xrange(27)])\n",
    "mat2 = np.array([[kl_prob(get_cond_dist(i), get_cond_dist(j)) for j in xrange(27)] for i in xrange(27)])\n",
    "mat2 = 1-np.array([[np.corrcoef(get_cond_dist(i), get_cond_dist(j))[0][1] for j in xrange(27)] for i in xrange(27)])\n",
    "ax = plt.imshow(mat2, cmap = 'gray', interpolation = 'nearest')\n",
    "plt.xticks(range(27), chars)\n",
    "plt.yticks(range(27), chars)\n",
    "plt.grid(False)\n",
    "plt.subplot(1,3,3)\n",
    "\n",
    "cos_dist = np.array([[sp.spatial.distance.cosine(b_ixn[i], b_ixn[j]) for j in xrange(27)] for i in xrange(27)])\n",
    "plt.imshow(-cos_dist, cmap = 'gray', interpolation = 'none')\n",
    "plt.xticks(range(27), chars)\n",
    "plt.yticks(range(27), chars)\n",
    "plt.grid(False)\n",
    "\n",
    "\n",
    "\n",
    "np.sum(mat2, axis = 0)\n",
    "plt.savefig(\"/tmp/bits_of_entropy.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "Zt9kSyZ4cwvb"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "CmXZDRqQwGXq"
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(20, 7))\n",
    "plt.subplot(2,1,1)\n",
    "q, r = np.linalg.qr(out_inn)\n",
    "\n",
    "#print(np.dot(np.transpose(q),q ))\n",
    "\n",
    "#print(abs((out_inn - q.dot(r)))).mean()\n",
    "\n",
    "plt.imshow(np.transpose(q),interpolation='none', cmap = 'bwr')\n",
    "#plt.imshow((r),interpolation='none', cmap = 'bwr')\n",
    "#plt.imshow((r),interpolation='none', cmap = 'bwr')\n",
    "plt.subplot(2,1,2)\n",
    "\n",
    "plt.plot(r.diagonal())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "y30RXjalnJK2"
   },
   "outputs": [],
   "source": [
    "\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "MHV903_mpA2L"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "0vB78tcLFf_7"
   },
   "outputs": [],
   "source": [
    "\n",
    "#What is the norm of the carried forward part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "hP-bFZWLGSB-"
   },
   "outputs": [],
   "source": [
    "norms = ([np.linalg.norm(df_states_pca[\"state\"][i]) for i in xrange(len(df_states_pca))])\n",
    "p = plt.hist(norms, bins = 100)\n",
    "print random_states_pca.shape\n",
    "print np.mean(norms)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "zLApEcZCvaQF"
   },
   "outputs": [],
   "source": [
    "def scatter_plot(pd_input, color_map, do_pca = True):\n",
    "  points = np.array([v[0] for v in pd_input.values])\n",
    "  if do_pca:\n",
    "    points = pca.transform(points)\n",
    "  ax = plt.scatter(points[:,0], points[:,1], c = color_map, cmap = 'bwr' )\n",
    "  return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "IrT2L5C_cK0h"
   },
   "outputs": [],
   "source": [
    "df_limit_history[\"state\"][0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "dhZK8O96ly1G"
   },
   "outputs": [],
   "source": [
    "#PCA K-means plot using random states generated in the PCA dimension for the kmeans\n",
    "k_means_dim = 1000\n",
    "kmean = KMeans(k_means_dim, precompute_distances = False)\n",
    "kmean.fit(random_states_pca)\n",
    "\n",
    "max_history = 5\n",
    "\n",
    "# #define the inputs\n",
    "# index = range(n_random) \n",
    "# np.random.shuffle(index)\n",
    "# inputs = np.zeros((27*num_each_char,27))\n",
    "# for i in range(27*num_each_char):\n",
    "#   inputs[i, i % 27] = 1\n",
    "\n",
    "# This produces n_random hidden states by running the RNN in feed forward mode with the input. We could add a random sampling probability here.\n",
    "df_states_kmeans = pd.DataFrame()\n",
    "errors = np.zeros(50)\n",
    "#h  = np.zeros((1, num_nodes))\n",
    "h  = sess.run(source_rnn_cell_pca.zero_state(1, dtype=tf.float32))\n",
    "\n",
    "losses = []\n",
    "\n",
    "for i in range(30000,30000+5000):\n",
    "  if i % 50 == 0:\n",
    "    h  = sess.run(source_rnn_cell_pca.zero_state(1, dtype=tf.float32))\n",
    "  h_in = h\n",
    "  h = np.reshape(kmean.cluster_centers_[kmean.predict(h_in)[0]], [1,-1])\n",
    "  h,loss = sess.run( [hidden_state_source_pca, train_loss_pca], feed_dict={input_placeholder:x[i], state_placeholder_pca:h, target_placeholder:y[i]})\n",
    "\n",
    "  d = {\"index\" : i, \"state\": h, \"input\":np.argmax(x[i]), \"state_in\":h_in, \"delta\":h-h_in, \"dist\": i % 50, \"loss\": loss, \"history\":history[:] }\n",
    "  df_states_kmeans = df_states_kmeans.append(d, ignore_index=True)\n",
    "  if i % 50 > 13:\n",
    "    losses.append(loss)\n",
    "  errors\n",
    "print(\"produced {} random states\".format(n_random))\n",
    "print(np.mean(losses))\n",
    "print len(df_states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "YW2Bl2XepLg-"
   },
   "outputs": [],
   "source": [
    "#Transition matrix for all character input. Note that space bar almost resets. But not quite :)\n",
    "fig = plt.figure(figsize=(12, 10))\n",
    "\n",
    "inputs = np.zeros((27,27))\n",
    "trans_x = []\n",
    "trans_y = []\n",
    "col = []\n",
    "for i in range(27):\n",
    "  inputs[i, i % 27] = 1\n",
    "transition_matrix = np.zeros((27, k_means_dim,k_means_dim))\n",
    "state_in = np.zeros((27, num_nodes))\n",
    "for i in range(k_means_dim):\n",
    "  for k in range(27):\n",
    "    state_in[k] =  kmean.cluster_centers_[i]\n",
    "  h_out = sess.run( [hidden_state_source_pca], feed_dict={input_placeholder:inputs, state_placeholder_pca:state_in})\n",
    "  h_out_pred  = kmean.predict(h_out[0])\n",
    "  for in_val in range(27):\n",
    "    transition_matrix[k,i,h_out_pred[in_val]] = 1\n",
    "    trans_x.append(h_out_pred[in_val])\n",
    "    trans_y.append(i)\n",
    "    col.append(in_val)\n",
    "y_ax = plt.subplot(1,1,1)    \n",
    "ax = plt.scatter(trans_x, trans_y, c = col, lw = 0,cmap = 'bwr', alpha = 0.5 ) \n",
    "cbar = fig.colorbar(ax)\n",
    "y_ax.set_xlim(0, k_means_dim)\n",
    "y_ax.set_ylim(0, k_means_dim)\n",
    "\n",
    "plt.savefig(\"/tmp/finite_state_machine_transition_matrix.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "9cCiKNHJrqEo"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "MYtvklCssX8c"
   },
   "outputs": [],
   "source": [
    "#tytyt\n",
    "ax = scatter_plot(df_states_pca[\"state\"],df_states_pca[\"input\"], False)\n",
    "x_map = []\n",
    "y_map = []\n",
    "fix_vects = np.zeros((27, pca_dim))\n",
    "for i in range(27):\n",
    "  w_i = w_h_pca_nxn[i] \n",
    "  b_i = b_h_pca[i]\n",
    "  _, vec = np.linalg.eig(w_i)\n",
    "  fix = np.dot(np.linalg.inv(np.eye(pca_dim)-w_i),b_i)\n",
    "  x_map.append(fix[0])\n",
    "  y_map.append( fix[1])\n",
    "  fix_vects[i] = fix\n",
    "  \n",
    "plt.scatter(x_map, y_map, c = np.arange(27),  cmap = 'bwr', s = 80)\n",
    "\n",
    "inputs_t = np.zeros((1,27))\n",
    "inputs_t[0,-1]=1\n",
    "h = sess.run( [hidden_state_source_pca], feed_dict={input_placeholder:inputs_t, state_placeholder_pca:np.reshape(fix, [1,-1])})\n",
    "print(np.sum(h-fix)**2)\n",
    "\n",
    "plt.savefig(\"/tmp/all_fixed_points_in_pca.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "ZIkkMHBQ5TDB"
   },
   "outputs": [],
   "source": [
    "#WARNING _ THIS IS PRETTY USELESS AS FAR AS I CAN TELL\n",
    "#Free form sampling Starting at Fixed points\n",
    "df_states_sampled = pd.DataFrame()\n",
    "#h  = np.zeros((1, num_nodes))\n",
    "h  = sess.run(source_rnn_cell.zero_state(1, dtype=tf.float32))\n",
    "losses = []\n",
    "input_t_saved = np.zeros((27, 27))\n",
    "\n",
    "for i in range(27):\n",
    "  input_t_saved[i, i] = 1\n",
    "samples = []\n",
    "for i in range(100):\n",
    "  in_real = np.zeros((1,27))\n",
    "  if i % 100 == 0:\n",
    "    h  = fix_vects\n",
    "    input_t = input_t_saved\n",
    "#   if i % 100  < 50 : #50 == 0 or i < 50:\n",
    "#     input_t = x[i]\n",
    "#     in_real = x[i]\n",
    "  out_eval, h, t = sess.run( [output_eval_pca, hidden_state_source_pca, output_source_pca], feed_dict={input_placeholder:input_t, state_placeholder_pca:h, target_placeholder:y[i], temperature: [2]})\n",
    "  input_t = np.zeros((27, 27))\n",
    "  for i in range(27):  \n",
    "    choice = np.random.choice(len(out_eval[i,:]), p=out_eval[i])\n",
    "    input_t[i, choice] = 1\n",
    "\n",
    "  d = {\"index\" : i, \"state\": h, \"input\":np.argmax(x[i]), \"output\" : t, \"input_vec\":(in_real)}\n",
    "  df_states_sampled = df_states_sampled.append(d, ignore_index=True)\n",
    "  if i % 50 > 13:\n",
    "    losses.append(loss)\n",
    "print(\"produced {} random states\".format(n_random))\n",
    "print(np.mean(losses))\n",
    "\n",
    "#train_task.print_text_input_output_targets(df_states_sampled[\"input_vec\"].values, df_states_sampled[\"output\"].values, df_states_sampled[\"output\"].values)\n",
    "\n",
    "\n",
    "for i in range(27):\n",
    "  vals = []\n",
    "  for k in  range(100):\n",
    "    vals.append(df_states_sampled[\"output\"][k][i:i+1])\n",
    "  print chars[i], 'text', train_task.print_text_input_output_targets(df_states_sampled[\"input_vec\"].values, df_states_sampled[\"input_vec\"].values,vals)\n",
    "\n",
    "    \n",
    "      \n",
    "      \n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "LkFSV_jjFT3t"
   },
   "outputs": [],
   "source": [
    "#Fixed points for different charactes.\n",
    "\n",
    "corr = np.corrcoef(fix_vects)\n",
    "plt.subplot(1,4,1)\n",
    "plt.title(\"fixed point corr\")\n",
    "corr = np.corrcoef(fix_vects)\n",
    "plt.imshow(corr,cmap=\"gray\",vmin=-1,vmax=1, interpolation='none' )\n",
    "plt.subplot(1,4,2)\n",
    "plt.title(\"off set Corre\")\n",
    "corr = np.corrcoef(b_h_pca)\n",
    "plt.imshow(corr,cmap=\"gray\",vmin=-1,vmax=1, interpolation='none' )\n",
    "plt.subplot(1,4,3)\n",
    "corr = np.corrcoef(np.transpose(b_h_pca))\n",
    "x_ = plt.imshow(corr,cmap=\"gray\",vmin=-1,vmax=1, interpolation='none' )\n",
    "# summed = np.sum(corr**4,1)\n",
    "# indx= np.argsort( summed)\n",
    "# corr_2= np.corrcoef(np.transpose(b_h_pca[:,indx]))\n",
    "\n",
    "plt.subplot(1,4,4)\n",
    "#correlation of the read out matrix\n",
    "#np.linalg.qr()\n",
    "\n",
    "# all_mat = np.zeros((k_means_dim, k_means_dim))\n",
    "# for i in range(27):\n",
    "#   all_mat += transition_matrix[i]\n",
    "# plt.imshow(all_mat,cmap=\"gray\")\n",
    "\n",
    "corr = np.corrcoef(np.transpose(out_inn))\n",
    "_=plt.imshow(corr,cmap=\"gray\",vmin=0,vmax=1, interpolation='none' )\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "vjR33txBHZm1"
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(10, 7))\n",
    "plt.subplot(2,3,1)\n",
    "_ =plt.plot((b_h_pca))\n",
    "\n",
    "\n",
    "# w_h_pca = np.zeros((27, pca_dim**2))\n",
    "# b_h_pca = np.zeros((27, pca_dim))\n",
    "# out_pca = np.dot( (pca.components_), out_inn)\n",
    "# init_pca = np.dot( (pca.components_), np.transpose(init))\n",
    "plt.subplot(2,3,2)\n",
    "plt.scatter(b_h_pca[:,3],b_h_pca[:,4], color= 'r')\n",
    "plt.scatter(b_h_pca[:,5],b_h_pca[:,6],color = 'b')\n",
    "\n",
    "plt.subplot(2,3,3)\n",
    "\n",
    "\n",
    "plt.subplot(2,3,4)\n",
    "_ =plt.plot(np.transpose(b_h_pca)[0:20,:])\n",
    "\n",
    "plt.subplot(2,3,5)\n",
    "_ =plt.plot(np.transpose(b_ixn))\n",
    "\n",
    "\n",
    "plt.subplot(2,3,6)\n",
    "_ =plt.plot(np.log(np.transpose(b_h_pca)[0:20,:]**2))\n",
    "\n",
    "plt.savefig(\"/tmp/draft_plot_overview2.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "1ozYiPRo2kW-"
   },
   "outputs": [],
   "source": [
    "plt.plot(gru308_ev, label = \"gru-308\")\n",
    "plt.plot(gru145_ev, label = \"gru-145\")\n",
    "plt.plot(rnn256_ev, label = \"rnn-256\")\n",
    "plt.plot(pca.explained_variance_ratio_, label = \"sls-216\")\n",
    "plt.ylim(1e-3, .2)\n",
    "plt.xlim(0, 100)\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "DKfZPLnt09HV"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "u5kSck29O4qY"
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(24, 7))\n",
    "\n",
    "plt.subplot(1,3,1)\n",
    "x_map = []\n",
    "y_map = []\n",
    "input_val = []\n",
    "input_val_2 = []\n",
    "all_ev  = []\n",
    "all_all_ev = []\n",
    "for k in range(27):\n",
    "  jacob = w_h_pca_nxn[k]\n",
    "  ev = np.linalg.eig(jacob)\n",
    "  all_all_ev.append(ev[1])\n",
    "  for j in range(num_nodes):\n",
    "    x_map.append(ev[0][j].real)\n",
    "    y_map.append(ev[0][j].imag)  \n",
    "    input_val.append(j)\n",
    "    input_val_2.append(k)\n",
    "    if j < 10:\n",
    "      all_ev.append(ev[1][:][j])\n",
    "\n",
    "\n",
    "\n",
    "ax = plt.scatter(x_map, y_map, c = input_val,  cmap = 'bwr', s = 20,  lw = 0)\n",
    "cbar = fig.colorbar(ax)\n",
    "\n",
    "plt.subplot(1,3,2)\n",
    "\n",
    "ax = plt.scatter(x_map, y_map, c = input_val_2,  cmap = 'bwr', s = 20,  lw = 0, alpha = 0.5)\n",
    "cbar = fig.colorbar(ax)\n",
    "\n",
    "plt.subplot(1,3,3)\n",
    "corr = np.corrcoef(all_ev)\n",
    "plt.imshow(np.abs(corr),cmap=\"gray\",vmin=0,vmax=0.5)#, interpolation='none' )\n",
    "\n",
    "plt.savefig(\"/tmp/spectrum_plot.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "ACAFWCtbQiUR"
   },
   "outputs": [],
   "source": [
    "print pca.explained_variance_ratio_[:10]\n",
    "print np.sum(pca.explained_variance_ratio_)\n",
    "\n",
    "vals = np.zeros((len(df_limit_history),max_history))\n",
    "for i in range(len(df_limit_history)):\n",
    "  for j in range(max_history):\n",
    "    vals[i,j] = np.linalg.norm(df_limit_history[\"history\"][i][j])  \n",
    "\n",
    "# x_val_decay = []\n",
    "# y_val_decay = []\n",
    "# vals_y = np.log(np.mean(vals,0))\n",
    "# for i in range(len(vals_y)):\n",
    "#   x_val_decay.append( np.log(i+1))\n",
    "#   y_val_decay.append(vals_y[i])\n",
    "#plt.subplot(1,2,1)\n",
    "#plt.plot(x_val, y_val)\n",
    "#plt.subplot(1,2,2)\n",
    "#plt.plot(np.mean(vals,0),hold = True)\n",
    "#plt.title(\"Norm of incoming state contribution after n steps\")\n",
    "\n",
    "np.sum(np.mean(vals,0))\n",
    "\n",
    "fig = plt.figure(figsize=(22, 5))\n",
    "\n",
    "x_val = []\n",
    "y_val = []\n",
    "col = []\n",
    "count = 0\n",
    "for k in range(27):\n",
    "  for j in range(num_nodes):\n",
    "    y_val.append( ((x_map[count]**2 + y_map[count]**2))**0.5)\n",
    "    x_val.append(j)\n",
    "    col.append(k)\n",
    "    count = count + 1\n",
    "plt.subplot(1,5,1)\n",
    "plt.title(\"Norm of Eigenvalues\")\n",
    "ax = plt.scatter(x_val, y_val, c = col,  cmap = 'bwr', s = 10,  lw = 0)\n",
    "cbar = fig.colorbar(ax)\n",
    "\n",
    "norms = []\n",
    "for k in range(27):\n",
    "  norms.append(np.linalg.norm(w_h_pca_nxn[k]))\n",
    "plt.subplot(1,5,2)\n",
    "plt.title(\"Norm of Weightmatrices\")\n",
    "\n",
    "plt.plot(norms)\n",
    "\n",
    "plt.subplot(1,5,3)\n",
    "plt.title(\"Decay of norm over time\")\n",
    "#plt.plot(x_val_decay, y_val_decay)\n",
    "plt.plot(np.mean(vals,0))\n",
    "# dets  = []\n",
    "# ranks = []\n",
    "# for k in range(27):\n",
    "#   dets.append(np.linalg.det(w_h_pca_nxn[k]))\n",
    "#   ranks.append(np.linalg.matrix_rank(w_h_pca_nxn[k]))\n",
    "# plt.plot(ranks)\n",
    "# plt.plot(dets)\n",
    "\n",
    "\n",
    "plt.subplot(1,5,4)\n",
    "plt.title(\"First 20 dimensions of 5 EV\")\n",
    "dets  = []\n",
    "for k in range(27):\n",
    "  plt.plot(all_all_ev[k][:20,:5],hold = True)\n",
    "  \n",
    "plt.subplot(1,5,5)\n",
    "plt.title(\"Correlation for offset vectors\")\n",
    "\n",
    "dets  = []\n",
    "# for k in range(27):\n",
    "#   plt.plot(np.transpose(all_all_ev[k][:10,:10]),hold = True)\n",
    "corr = np.corrcoef((b_h_pca))\n",
    "x_ = plt.imshow(corr,cmap=\"gray\",vmin=-1,vmax=1, interpolation='none' )\n",
    "\n",
    "plt.savefig(\"/tmp/eigen_val_and_decay.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "mdmtdf63fh5d"
   },
   "outputs": [],
   "source": [
    "\n",
    "fig = plt.figure(figsize=(22, 16))\n",
    "\n",
    "for i in range(27):\n",
    "  plt.subplot(5,6, i + 1)\n",
    "\n",
    "  #corr = np.abs(np.corrcoef(np.transpose(b_h_pca)[0:20,:],(all_all_ev[k][:20,:20]), 0))\n",
    "  #plt.imshow(corr,cmap=\"gray\",vmin=-1,vmax=1, interpolation='none') #\n",
    "\n",
    "  corr = np.abs(np.corrcoef(b_h_pca[:,:pca_dim],np.transpose(all_all_ev[i][:pca_dim,:30]), 1))\n",
    "  plt.imshow(corr,cmap=\"gray\",vmin=0,vmax=1, interpolation='none') #\n",
    "\n",
    "plt.subplot(5,6, 2)\n",
    "\n",
    "plt.title(\"correlation between offset vectors and 30 first eigenvectors\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "rYew2sYbirsD"
   },
   "outputs": [],
   "source": [
    "#Below here is random stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "KiqTdaPEq-Ov"
   },
   "outputs": [],
   "source": [
    "k =10\n",
    "matrx = w_h_pca_nxn[k]\n",
    "evals, evects = np.linalg.eig(matrx)\n",
    "ident = np.identity(num_nodes) \n",
    "\n",
    "eig_num = 30\n",
    "#print np.dot(matrx - evals[eig_num]* ident, (evects[:,eig_num])) [0:3]\n",
    "\n",
    "identity = np.identity(num_nodes)\n",
    "\n",
    "ident = np.diag(evals)\n",
    "  \n",
    "print (np.dot((evects), np.dot(ident, np.linalg.inv(evects))) - matrx) [0:2,0:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "kQevzh5r21jB"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "Qedc-8O_yXih"
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "fig = plt.figure(figsize=(8, 8))\n",
    "ax = scatter_plot(df_states[\"state\"], df_states[\"input\"])\n",
    "cbar = fig.colorbar(ax)\n",
    "\n",
    "plt.savefig(\"/tmp/pca_plot_hidden_state_by_input.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "lg1Wr5uszuNt"
   },
   "outputs": [],
   "source": [
    "kmean = KMeans(200)\n",
    "kmean.fit(random_states[:8000,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "3vdSkVULALMt"
   },
   "outputs": [],
   "source": [
    "kmean = KMeans(200)\n",
    "kmean.fit(random_states[:8000,:])\n",
    "\n",
    "max_history = 5\n",
    "\n",
    "#define the inputs\n",
    "index = range(n_random) \n",
    "np.random.shuffle(index)\n",
    "inputs = np.zeros((27*num_each_char,27))\n",
    "for i in range(27*num_each_char):\n",
    "  inputs[i, i % 27] = 1\n",
    "\n",
    "# This produces n_random hidden states by running the RNN in feed forward mode with the input. We could add a random sampling probability here.\n",
    "df_states = pd.DataFrame()\n",
    "errors = np.zeros(50)\n",
    "#h  = np.zeros((1, num_nodes))\n",
    "h  = sess.run(source_rnn_cell.zero_state(1, dtype=tf.float32))\n",
    "\n",
    "losses = []\n",
    "\n",
    "for i in range(10000,10000+5000):\n",
    "  if i % 50 == 0:\n",
    "    h  = sess.run(source_rnn_cell.zero_state(1, dtype=tf.float32))\n",
    "  h_in = h\n",
    "  h = np.reshape(kmean.cluster_centers_[kmean.predict(h_in)[0]], [1,-1])\n",
    "  h,loss = sess.run( [hidden_state_source, train_loss], feed_dict={input_placeholder:x[i], state_placeholder:h, target_placeholder:y[i]})\n",
    "\n",
    "  d = {\"index\" : i, \"state\": h, \"input\":np.argmax(x[i]), \"state_in\":h_in, \"delta\":h-h_in, \"dist\": i % 50, \"loss\": loss, \"history\":history[:] }\n",
    "  df_states = df_states.append(d, ignore_index=True)\n",
    "  if i % 50 > 13:\n",
    "    losses.append(loss)\n",
    "  errors\n",
    "print(\"produced {} random states\".format(n_random))\n",
    "print(np.mean(losses))\n",
    "print len(df_states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "WP76nbLFFt7E"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "PfDiqMpd9w2p"
   },
   "outputs": [],
   "source": [
    "#kmean.predict(random_states[:200,:])\n",
    "data = pca.transform(kmean.cluster_centers_)\n",
    "plt.scatter(data[:,0], data[:,1])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "XPH8AOMjgYzc"
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "from sklearn.manifold import TSNE\n",
    "def plot_tsne(data_points, labels):\n",
    "  \n",
    "  model = TSNE(n_components=2, random_state=0)\n",
    "  np.set_printoptions(suppress=True)\n",
    "  fit_tsne = model.fit_transform(data_points)\n",
    "\n",
    "  def map_vals(vals):\n",
    "    vals_set = list(set(vals))\n",
    "    map_dict = {}\n",
    "    for i, v in enumerate(vals_set):\n",
    "      map_dict[v] = i\n",
    "    return [map_dict[v] for v in vals]\n",
    "\n",
    "  #print fit_tsne\n",
    "\n",
    "  x = [h[0] for h in fit_tsne]\n",
    "  y = [h[1] for h in fit_tsne]\n",
    "\n",
    "  colors = map_vals(labels)\n",
    "  print len(set(colors)) \n",
    "\n",
    "  p = plt.scatter(x + np.random.randn(len(x))*0.01,y,c = colors, cmap = \"Set1\")\n",
    "  return x, y, colors\n",
    "  \n",
    "  \n",
    "x_val, y_val, col = plot_tsne(random_states, df_states[\"input\"])\n",
    "  \n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "kG2VX7anY8WF"
   },
   "outputs": [],
   "source": [
    "\n",
    "#Free form sampling\n",
    "df_states_sampled = pd.DataFrame()\n",
    "#h  = np.zeros((1, num_nodes))\n",
    "h  = sess.run(source_rnn_cell.zero_state(1, dtype=tf.float32))\n",
    "losses = []\n",
    "\n",
    "samples = []\n",
    "for k in range(1000):\n",
    "  i = (k // 100) * 4 + k\n",
    "  in_real = np.zeros((1,27))\n",
    "  if i % 100 == 0:\n",
    "    h  = sess.run(source_rnn_cell.zero_state(1, dtype=tf.float32))\n",
    "  if i % 100  < 50 : #50 == 0 or i < 50:\n",
    "    input_t = x[i]\n",
    "    in_real = x[i]\n",
    "  out_eval, h,loss, t = sess.run( [output_eval, hidden_state_source, train_loss, output_source], feed_dict={input_placeholder:input_t, state_placeholder:h, target_placeholder:y[i], temperature: [10]})\n",
    "  #out_eval[0][-1]= 1 -np.sum(out_eval[0][0:-1])\n",
    "  #input_t = np.random.multinomial(1, out_eval, size=1)\n",
    "  choice = np.random.choice(len(out_eval[0,:]), p=out_eval[0])\n",
    "  input_t = np.zeros((1, 27))\n",
    "  input_t[0, choice] = 1\n",
    "\n",
    "  d = {\"index\" : i, \"state\": h, \"input\":np.argmax(x[i]), \"output\" : t, \"input_vec\":(in_real)}\n",
    "  df_states_sampled = df_states_sampled.append(d, ignore_index=True)\n",
    "  if i % 50 > 13:\n",
    "    losses.append(loss)\n",
    "print(\"produced {} random states\".format(n_random))\n",
    "print(np.mean(losses))\n",
    "\n",
    "train_task.print_text_input_output_targets(df_states_sampled[\"input_vec\"].values, df_states_sampled[\"output\"].values, df_states_sampled[\"output\"].values)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "peYqIRNIho_0"
   },
   "outputs": [],
   "source": [
    "pca = PCA(n_components=2)\n",
    "pca.fit(random_states)\n",
    "\n",
    "stats ={}\n",
    "stats['pca_explained_variance'] = pca.explained_variance_ratio_\n",
    "print(stats)\n",
    "\n",
    "\n",
    "#fixed_point_states = np.array([v for v in df_expansions[\"state\"].values])\n",
    "#approx_out_pca = pca.transform(fixed_point_states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "i9u4dGYxzBam"
   },
   "outputs": [],
   "source": [
    "def get_energy(h_in, input_i, componentwise = False):\n",
    "  inputs = np.zeros((1,27))\n",
    "  h_in = np.reshape(h_in, [1, -1])\n",
    "  inputs[0,input_i] = 1\n",
    "  h = sess.run( hidden_state, feed_dict={input_placeholder:inputs, state_placeholder:h_in, expansion_point:h_in})\n",
    "  coer = np.corrcoef(h, h_in)[1,0]\n",
    "  if componentwise:\n",
    "    return (h_in-h)**2, coer\n",
    "  else:\n",
    "    return (np.sum((h_in-h)**2, 1)), coer\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "sViicbVonYTZ"
   },
   "outputs": [],
   "source": [
    "pca = PCA(n_components=2)\n",
    "pca.fit(random_states)\n",
    "#plt.plot(random_states[0:100,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "cqHCxMbiJ7GY"
   },
   "outputs": [],
   "source": [
    "pca = PCA(n_components=2)\n",
    "pca.fit(random_states)\n",
    "\n",
    "fixed_point_states = np.array([v for v in df_expansions[\"state\"].values])\n",
    "approx_out_pca = pca.transform(fixed_point_states)\n",
    "\n",
    "\n",
    "save_fig_path = \"/tmp/\"\n",
    "fig = plt.figure(figsize=(22, 7))\n",
    "bounds =range(len(df_expansions))\n",
    "bounds = list(np.where(df_expansions[\"energy\"][:] < 0.01))[0]\n",
    "print len(bounds)\n",
    "plots = []\n",
    "plots.append(plt.subplot(2,3,1))\n",
    "cax = plt.scatter(approx_out_pca[bounds,0],approx_out_pca[bounds,1], c = np.log(df_expansions[\"energy\"][bounds])/np.log(10), cmap = \"autumn\" )\n",
    "cbar = fig.colorbar(cax)\n",
    "\n",
    "norm_states = np.zeros(len(random_states))\n",
    "for i, state in enumerate( random_states):\n",
    "  norm_states[i],_ = get_energy(np.reshape(state,[1,-1]), np.argmax(x[i]))\n",
    "  \n",
    "plots.append(plt.subplot(2,3,2))\n",
    "\n",
    "all_states_pca = pca.transform(random_states)  \n",
    "cax = plt.scatter(all_states_pca[:,0],all_states_pca[:,1], c = np.log(norm_states)/np.log(10), cmap = \"autumn\", hold = True )\n",
    "cbar = fig.colorbar(cax) \n",
    "\n",
    "uniform_states =  np.zeros((len(random_states),num_nodes))\n",
    "norm_states_uniform = np.zeros(len(random_states))\n",
    "for i, state in enumerate( random_states):\n",
    "  state_new = np.reshape( np.random.uniform(-1, 1, num_nodes), [1, -1])\n",
    "  norm_states_uniform[i] = get_energy(state_new, np.argmax(x[i]))[0]\n",
    "  uniform_states[i,:]=(state_new)[0]\n",
    "\n",
    "\n",
    "plots.append(plt.subplot(2,3,3))\n",
    "uniform_states_pca = pca.transform(uniform_states)  \n",
    "cax = plt.scatter(uniform_states_pca[:,0],uniform_states_pca[:,1], c = np.log(norm_states_uniform)/np.log(10), cmap = \"autumn\", hold = True )\n",
    "cbar = fig.colorbar(cax) \n",
    "\n",
    "# norm = np.zeros(len(df_new[\"state\"]))\n",
    "# for i, state in enumerate( df_new[\"state\"]):\n",
    "#   norm[i] =sum(state**2)**0.5\n",
    "\n",
    "# cax = plt.scatter(approx_out_pca[bounds,0],approx_out_pca[bounds,1], c = norm[bounds], cmap = \"autumn\" )\n",
    "# cbar = fig.colorbar(cax)\n",
    "\n",
    "# plt.subplot(2,2,1)\n",
    "\n",
    "\n",
    "# # all_states_pca = pca.transform(random_states)  \n",
    "# # cax = plt.scatter(all_states_pca[:,0],all_states_pca[:,1], c = norm_states, cmap = \"autumn\", hold = True )\n",
    "# # cbar = fig.colorbar(cax) \n",
    "# for i, plot in enumerate(plots):\n",
    "#   plot.set_xlim([-8,6])\n",
    "#   plot.set_ylim([-8,6])\n",
    "  \n",
    "save_fig_path + \"overview_plt\"\n",
    "\n",
    "plt.savefig(save_fig_path + \"overview_plt\" + FLAGS.source_rnn + \"_num_\" + str(num) + \"_reg_\" + reg + snapshop +  \".pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "TstXgBOmybhi"
   },
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "  return 1/(1+np.exp(-x))\n",
    "\n",
    "#This might be a good plot - u and r by energy for fixed point and samples points\n",
    "# h_t_bxn = u_bxn * h_tm1_bxn  + (1.0-u_bxn) * c_t_bxn\n",
    "pre_activations = source_rnn_cell._pre_activations\n",
    "mean_act = []\n",
    "energy  = []\n",
    "mean_r = []\n",
    "bounds = list(np.where(df_expansions[\"energy\"][:] < 0.001))[0]\n",
    "number_to_plot = 10000\n",
    "for k in range(number_to_plot):\n",
    "  i = bounds[index[k] % len(bounds)]\n",
    "  state  = fixed_point_states[i:i+1,:]\n",
    "  input_val = df_expansions[\"input\"][i]\n",
    "  inputs = np.zeros((1,27))\n",
    "  inputs[0,input_val]=1\n",
    "  tensors = [pre_activations[\"fg_bias\"], pre_activations[\"xh_t_bxipn\"],pre_activations[\"r_act_bxn\"],pre_activations[\"u_act_bxn\"],pre_activations[\"xrh_t_bxipn\"], pre_activations[\"c_act_bxn\"]]\n",
    "  tensors_eval = sess.run(tensors,  {input_placeholder:inputs, state_placeholder: state})\n",
    "  mean_act.append(np.mean(sigmoid(tensors_eval[3][0]+forget_gate_hack)))\n",
    "  mean_r.append(np.mean(sigmoid(tensors_eval[2][0])))  \n",
    "  energy.append(df_expansions[\"energy\"][i])\n",
    "\n",
    "fig = plt.figure(figsize=(22, 7))\n",
    "\n",
    "plots = []\n",
    "plots.append(plt.subplot(1,3,1))\n",
    "plt.title(\"Fixed points\")\n",
    "plt.xlabel(\"mean act u\")\n",
    "plt.ylabel(\"mean act r\")\n",
    "\n",
    "cax = plt.scatter(mean_act,mean_r, c = np.log(energy)/np.log(10),alpha = 0.2, cmap = \"autumn\",lw = 0  )\n",
    "cbar = fig.colorbar(cax) \n",
    "\n",
    "\n",
    "mean_act_states = []\n",
    "mean_energy_states  = []\n",
    "mean_r_states = []\n",
    "\n",
    "for i, state in enumerate(random_states):\n",
    "  inputs = np.zeros((1,27))\n",
    "  input_val = np.argmax(x[i])\n",
    "  inputs[0,input_val]=1\n",
    "  state  = random_states[i:i+1,:]\n",
    "  tensors = [pre_activations[\"fg_bias\"], pre_activations[\"xh_t_bxipn\"],pre_activations[\"r_act_bxn\"],pre_activations[\"u_act_bxn\"],pre_activations[\"xrh_t_bxipn\"], pre_activations[\"c_act_bxn\"]]\n",
    "  tensors_eval = sess.run(tensors,  {input_placeholder:inputs, state_placeholder: state})\n",
    "  mean_act_states.append(np.mean(sigmoid(tensors_eval[3][0]+forget_gate_hack)))\n",
    "  mean_r_states.append(np.mean(sigmoid(tensors_eval[2][0])))\n",
    "  energy_new, _ = get_energy( state, input_val)\n",
    "  mean_energy_states.append(energy_new)\n",
    "  \n",
    "plots.append(plt.subplot(1,3,2))\n",
    "cax = plt.scatter(mean_act_states,mean_r_states, c = np.log(mean_energy_states)/np.log(10),alpha = 0.2, cmap = \"autumn\" ,lw = 0 )\n",
    "cbar = fig.colorbar(cax) \n",
    "\n",
    "plt.title(\"Random States from trajectory\")\n",
    "plt.xlabel(\"mean act u\")\n",
    "plt.ylabel(\"mean act r\")\n",
    "\n",
    "mean_act_states = []\n",
    "mean_energy_states  = []\n",
    "mean_r_states = []\n",
    "\n",
    "for i, state in enumerate(random_states):\n",
    "  inputs = np.zeros((1,27))\n",
    "  input_val = np.argmax(x[i])\n",
    "  inputs[0,input_val]=1\n",
    "  state  = np.reshape( np.random.uniform(-1, 1, num_nodes), [1, -1])\n",
    "  tensors = [pre_activations[\"fg_bias\"], pre_activations[\"xh_t_bxipn\"],pre_activations[\"r_act_bxn\"],pre_activations[\"u_act_bxn\"],pre_activations[\"xrh_t_bxipn\"], pre_activations[\"c_act_bxn\"]]\n",
    "  tensors_eval = sess.run(tensors,  {input_placeholder:inputs, state_placeholder: state})\n",
    "  mean_act_states.append(np.mean(sigmoid(tensors_eval[3][0]+forget_gate_hack)))\n",
    "  mean_r_states.append(np.mean(sigmoid(tensors_eval[2][0])))\n",
    "  energy_new, _ = get_energy( state, input_val)\n",
    "  mean_energy_states.append(energy_new)\n",
    "  \n",
    "plots.append(plt.subplot(1,3,3))\n",
    "cax = plt.scatter(mean_act_states,mean_r_states, c = np.log(mean_energy_states)/np.log(10),alpha = 0.2, cmap = \"autumn\" ,lw = 0 )\n",
    "cbar = fig.colorbar(cax) \n",
    "\n",
    "plt.title(\"Random States from all space\")\n",
    "plt.xlabel(\"mean act u\")\n",
    "plt.ylabel(\"mean act r\")\n",
    "for i, plot in enumerate(plots):\n",
    "  plot.set_xlim([0,1])\n",
    "  plot.set_ylim([0,1])\n",
    "\n",
    "  \n",
    "  \n",
    "plt.savefig(save_fig_path + \"overview_plt\" + FLAGS.source_rnn + \"_num_\" + str(num) + \"_reg_\" + reg + snapshop +  \".pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "PJb-kxslYdMK"
   },
   "outputs": [],
   "source": [
    "forget_gate_hack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "ijghs_Bi8Lfo"
   },
   "outputs": [],
   "source": [
    "mean_act_states = []\n",
    "mean_energy_states  = []\n",
    "mean_r_states = []\n",
    "number_to_plot = 1000\n",
    "plots = []\n",
    "for k in range(number_to_plot):\n",
    "  i = index[k]%len(df_expansions[\"input\"])\n",
    "  inputs = np.zeros((1,27))\n",
    "  input_val = df_expansions[\"input\"][i]\n",
    "  inputs[0,input_val]=1\n",
    "  state  = fixed_point_states[i:i+1,:]\n",
    "\n",
    "  tensors = [pre_activations[\"fg_bias\"], pre_activations[\"xh_t_bxipn\"],pre_activations[\"r_act_bxn\"],pre_activations[\"u_act_bxn\"],pre_activations[\"xrh_t_bxipn\"], pre_activations[\"c_act_bxn\"]]\n",
    "  tensors_eval = sess.run(tensors,  {input_placeholder:inputs, state_placeholder: state})\n",
    "  energy_new, _ = get_energy( state, input_val, componentwise = componentwise)\n",
    "  for k in range(num_nodes):\n",
    "    mean_energy_states.append(energy_new[0][k % len(energy_new[0])])\n",
    "    mean_act_states.append((tensors_eval[3][0]+6)[k])\n",
    "    mean_r_states.append(k)  \n",
    "plots.append(plt.subplot(1,3,1) )   \n",
    "cax = plt.scatter(mean_r_states, mean_act_states, c = np.log(mean_energy_states)/np.log(10),alpha =0.1, cmap = \"autumn\",lw = 0)  \n",
    "cbar = fig.colorbar(cax) \n",
    "\n",
    "mean_act_states = []\n",
    "mean_energy_states  = []\n",
    "mean_r_states = []\n",
    "number_to_plot = 400\n",
    "for k in range(number_to_plot):\n",
    "  i = index[k]\n",
    "  inputs = np.zeros((1,27))\n",
    "  input_val = np.argmax(x[i])\n",
    "  inputs[0,input_val]=1\n",
    "  state  = np.reshape( np.random.uniform(-1, 1, num_nodes), [1, -1])\n",
    "  state  = random_states[i:i+1,:]\n",
    "\n",
    "  tensors = [pre_activations[\"fg_bias\"], pre_activations[\"xh_t_bxipn\"],pre_activations[\"r_act_bxn\"],pre_activations[\"u_act_bxn\"],pre_activations[\"xrh_t_bxipn\"], pre_activations[\"c_act_bxn\"]]\n",
    "  tensors_eval = sess.run(tensors,  {input_placeholder:inputs, state_placeholder: state})\n",
    "  energy_new, _ = get_energy( state, input_val, componentwise = componentwise)\n",
    "  for k in range(num_nodes):\n",
    "    mean_energy_states.append(energy_new[0][k % len(energy_new[0])])\n",
    "    mean_act_states.append((tensors_eval[3][0]+6)[k])\n",
    "    mean_r_states.append(k)  \n",
    "plots.append(plt.subplot(1,3,2))    \n",
    "cax = plt.scatter(mean_r_states, mean_act_states, c = np.log(mean_energy_states)/np.log(10),alpha =0.1, cmap = \"autumn\",lw = 0)  \n",
    "cbar = fig.colorbar(cax) \n",
    "saved_u = mean_act_states\n",
    "\n",
    "mean_act_states = []\n",
    "mean_energy_states  = []\n",
    "mean_r_states = []\n",
    "number_to_plot = 400\n",
    "for k in range(number_to_plot):\n",
    "  i = index[k]\n",
    "  inputs = np.zeros((1,27))\n",
    "  input_val = np.argmax(x[i])\n",
    "  inputs[0,input_val]=1\n",
    "  state  = np.reshape( np.random.uniform(-1, 1, num_nodes), [1, -1])\n",
    "\n",
    "  tensors = [pre_activations[\"fg_bias\"], pre_activations[\"xh_t_bxipn\"],pre_activations[\"r_act_bxn\"],pre_activations[\"u_act_bxn\"],pre_activations[\"xrh_t_bxipn\"], pre_activations[\"c_act_bxn\"]]\n",
    "  tensors_eval = sess.run(tensors,  {input_placeholder:inputs, state_placeholder: state})\n",
    "  energy_new, _ = get_energy( state, input_val, componentwise = componentwise)\n",
    "  for k in range(num_nodes):\n",
    "    mean_energy_states.append(energy_new[0][k % len(energy_new[0])])\n",
    "    mean_act_states.append((tensors_eval[3][0]+6)[k])\n",
    "    mean_r_states.append(k)  \n",
    "plots.append(plt.subplot(1,3,3))    \n",
    "cax = plt.scatter(mean_r_states, mean_act_states, c = np.log(mean_energy_states)/np.log(10),alpha =0.1, cmap = \"autumn\",lw = 0)  \n",
    "cbar = fig.colorbar(cax) \n",
    "\n",
    "#for i, plot in enumerate(plots):\n",
    "  #plot.set_xlim([0,num_nodes])\n",
    "  #plot.set_ylim([-20,20])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "oirhHcX3Ya_X"
   },
   "outputs": [],
   "source": [
    "bins_energy = np.arange(-80,45,0.1)\n",
    "p2 = plt.hist(mean_act_states, bins=bins_energy, alpha = 1) #includes everything. It's the incoming activation to the U gate. Random suff, -1 to 1 in each dimension\n",
    "p2 = plt.hist(saved_u, bins=bins_energy, alpha = 1, color = 'g') #includes everything. It's the incoming activation to the U gate. Random suff, -1 to 1 in each dimension\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "xh3j_HsQzJwn"
   },
   "outputs": [],
   "source": [
    "char = 1\n",
    "constant_input = np.zeros((batch_size_fp * 27,27))\n",
    "constant_input[:,char] = 1\n",
    "#expansion_point_vals = np.reshape(sess.run(fixed_point_rnn_cell.expansion_points),[ 270, -1])\n",
    "for i in range(10):\n",
    "  expansion_state_val = np.reshape(state_input,[1,-1])  # expansion_point_vals[i:i+1,:]\n",
    "  state = starting_states[i:i+1,:]\n",
    "  approx_out, true_out = sess.run([output, hidden_state_true], feed_dict={input_placeholder:constant_input[i:i+1,:], state_placeholder:state, expansion_point: expansion_state_val  })\n",
    "\n",
    "\n",
    "  pca = PCA(n_components=2)\n",
    "  pca.fit(random_states)\n",
    "  approx_out_pca = pca.transform(approx_out[0])\n",
    "  true_out_pca = pca.transform(true_out)\n",
    "\n",
    "  expansion_pca = pca.transform(expansion_state_val)\n",
    "  state_pca = pca.transform(state)\n",
    "\n",
    "  plt.scatter(approx_out_pca[:,0],approx_out_pca[:,1]) \n",
    "\n",
    "  plt.scatter(state_pca[:,0],state_pca[:,1], c = 'g', hold = True)\n",
    "  plt.scatter(true_out_pca[:,0],true_out_pca[:,1], c = 'r', hold = True) \n",
    "  plt.scatter(expansion_pca[:,0], expansion_pca[:,1], c = 'y', hold = True, s = 80)\n",
    "  plt.plot([expansion_pca[:,0],state_pca[:,0]], [expansion_pca[:,1],state_pca[:,1]], c = 'y', hold = True)\n",
    "  \n",
    "  plt.plot([state_pca[:,0],true_out_pca[:,0]],[state_pca[:,1],true_out_pca[:,1]], c = 'r', hold = True) \n",
    "  plt.plot([state_pca[:,0],approx_out_pca[:,0]],[state_pca[:,1],approx_out_pca[:,1]], c = 'b', hold = True, alpha = 0.5) \n",
    "  \n",
    "  \n",
    "  approx_out_pca = pca.transform(approx_out[1])\n",
    "  plt.scatter(approx_out_pca[:,0],approx_out_pca[:,1],c ='k',s = 160) \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "C50d-4Ymfj60"
   },
   "outputs": [],
   "source": [
    "char = 1 \n",
    "print df_states[df_states[\"input\"]==1][\"state\"][0:1]\n",
    "         \n",
    "print(df_expansions[\"energy\"].min())\n",
    "\n",
    "best = df_expansions[df_expansions[\"input\"] == 1][\"energy\"].argmin()\n",
    "tmp = df_expansions[df_expansions[\"input\"] == 1]\n",
    "\n",
    "print tmp[\"energy\"][best]\n",
    "state_input = np.reshape(tmp[\"state\"][best], [1,-1])\n",
    "print get_energy(state_input, 1)\n",
    "print best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "rD4GVRAFzD9S"
   },
   "outputs": [],
   "source": [
    "df_expansions = pd.read_pickle(\"/tmp/results_csv_huge_fp_new\" + FLAGS.source_rnn + \".pkl\")\n",
    "print len(df_expansions)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "ekKzEbfe9XkJ"
   },
   "outputs": [],
   "source": [
    "dist = []\n",
    "for k in range(5):\n",
    "  np.random.shuffle(index)\n",
    "  for m in range(4):\n",
    "    i = index[k]\n",
    "    state = random_states[i:i+1,:] \n",
    "    expansion_state_val = state + np.random.randn(1,num_nodes)*0.4\n",
    "    dist.append( np.sum((expansion_state_val-state)**2))\n",
    "    approx_out, true_out = sess.run([output, hidden_state_true], feed_dict={input_placeholder:x[i], state_placeholder:state, expansion_point: expansion_state_val  })\n",
    "\n",
    "\n",
    "    pca = PCA(n_components=2)\n",
    "    pca.fit(random_states)\n",
    "    approx_out_pca = pca.transform(approx_out[0])\n",
    "    true_out_pca = pca.transform(true_out)\n",
    "\n",
    "    expansion_pca = pca.transform(expansion_state_val)\n",
    "    state_pca = pca.transform(state)\n",
    "\n",
    "    plt.scatter(approx_out_pca[:,0],approx_out_pca[:,1], hold = True) \n",
    "\n",
    "    plt.scatter(state_pca[:,0],state_pca[:,1], c = 'g', hold = True)\n",
    "    plt.scatter(true_out_pca[:,0],true_out_pca[:,1], c = 'r', hold = True) \n",
    "    plt.scatter(expansion_pca[:,0], expansion_pca[:,1], c = 'y', hold = True, s = 50)\n",
    "    plt.plot([expansion_pca[:,0],state_pca[:,0]], [expansion_pca[:,1],state_pca[:,1]], c = 'y', hold = True)\n",
    "\n",
    "    plt.plot([state_pca[:,0],true_out_pca[:,0]],[state_pca[:,1],true_out_pca[:,1]], c = 'r', hold = True) \n",
    "    plt.plot([state_pca[:,0],approx_out_pca[:,0]],[state_pca[:,1],approx_out_pca[:,1]], c = 'b', hold = True, alpha = 0.5) \n",
    "\n",
    "\n",
    "print(np.mean(dist))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "4oYPiQAL8HE_"
   },
   "outputs": [],
   "source": [
    "df_distance_plot = pd.DataFrame()\n",
    "distances = np.arange(0,50,2)\n",
    "for d in distances:\n",
    "  np.random.shuffle(index)\n",
    "  for k in range(50):\n",
    "    i = index[k]\n",
    "    state = random_states[i:i+1,:] \n",
    "    expansion_state_val = state + np.random.randn(1,num_nodes)*((d)**0.5) /( num_nodes )**0.5\n",
    "    distance = np.sum((expansion_state_val-state)**2)\n",
    "    approx_out, true_out = sess.run([output, hidden_state_true], feed_dict={input_placeholder:x[i], state_placeholder:state, expansion_point: expansion_state_val })\n",
    "    cos_err = sp.spatial.distance.cosine(approx_out[0], true_out)\n",
    "    err = np.sum((true_out-approx_out[0])**2)\n",
    "    energy_expansion = np.sum((expansion_state_val-approx_out[1])**2)\n",
    "    energy_state = np.sum((state-true_out)**2)\n",
    "\n",
    "    d_entry = {\"cos_err\" : cos_err, \"err\": err, \"input\":np.argmax(x[i]), \"energy_expansion\":energy_expansion, \"distance\":distance, \"energy_state\":energy_state}\n",
    "    df_distance_plot = df_distance_plot.append(d_entry, ignore_index=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "_TsTXCNHA_XQ"
   },
   "outputs": [],
   "source": [
    "plt.subplot(1,2,1)\n",
    "plt.scatter(df_distance_plot['distance'], df_distance_plot['cos_err'],c =  np.log(df_distance_plot['energy_expansion']), cmap = 'autumn')\n",
    "\n",
    "plt.subplot(1,2,2)\n",
    "plt.scatter(df_distance_plot['distance'], df_distance_plot['energy_expansion'],c =  np.log(df_distance_plot['err']), cmap = 'autumn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "0LsBB6YOAECf"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "FZD9ugMIUodd"
   },
   "outputs": [],
   "source": [
    "# \n",
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "JSJs9xRrUDvr"
   },
   "outputs": [],
   "source": [
    "#componentwise plot of above - 1 point per dimension\n",
    "def add_np_arrays(x,y):\n",
    "  x = np.concatenate([x,y],1)\n",
    "\n",
    "pre_activations = source_rnn_cell._pre_activations\n",
    "mean_act = np.array([])\n",
    "energy  = np.array([])\n",
    "mean_r =  np.array([])\n",
    "inputs = np.zeros((1,27))\n",
    "inputs[0,0]=1\n",
    "for i, state in enumerate(res_states):\n",
    "  state  = res_states[i:i+1,:]\n",
    "  tensors = [pre_activations[\"fg_bias\"], pre_activations[\"xh_t_bxipn\"],pre_activations[\"r_act_bxn\"],pre_activations[\"u_act_bxn\"],pre_activations[\"xrh_t_bxipn\"], pre_activations[\"c_act_bxn\"]]\n",
    "  tensors_eval = sess.run(tensors,  {input_placeholder:inputs, state_placeholder: state})\n",
    "  add_np_arrays(mean_act, sigmoid(tensors_eval[3][0]))\n",
    "  add_np_arrays(mean_r,sigmoid(tensors_eval[2][0])) \n",
    "  add_np_arrays(energy,(get_energy(state,10, componentwise = True))\n",
    "\n",
    "fig = plt.figure(figsize=(18, 5))\n",
    "\n",
    "plots = []\n",
    "plots.append(plt.subplot(1,3,1))\n",
    "plt.title(\"Fixed points\")\n",
    "plt.xlabel(\"mean act u\")\n",
    "plt.ylabel(\"mean act r\")\n",
    "\n",
    "cax = plt.scatter(mean_act,mean_r, c = np.log(energy)/np.log(10),alpha = 0.2, cmap = \"autumn\"  )\n",
    "cbar = fig.colorbar(cax) \n",
    "\n",
    "\n",
    "mean_act_states = []\n",
    "mean_energy_states  = []\n",
    "mean_r_states = []\n",
    "\n",
    "for i, state in enumerate(random_states):\n",
    "  inputs = np.zeros((1,27))\n",
    "  input_val = np.argmax(x[i])\n",
    "  inputs[0,input_val]=1\n",
    "  state  = random_states[i:i+1,:]\n",
    "  tensors = [pre_activations[\"fg_bias\"], pre_activations[\"xh_t_bxipn\"],pre_activations[\"r_act_bxn\"],pre_activations[\"u_act_bxn\"],pre_activations[\"xrh_t_bxipn\"], pre_activations[\"c_act_bxn\"]]\n",
    "  tensors_eval = sess.run(tensors,  {input_placeholder:inputs, state_placeholder: state})\n",
    "  mean_act_states.append((sigmoid(tensors_eval[3][0])))\n",
    "  mean_r_states.append((sigmoid(tensors_eval[2][0])))\n",
    "  energy_new, _ = get_energy( state, input_val, componentwise = True)\n",
    "  mean_energy_states.append(energy_new)\n",
    "  \n",
    "plots.append(plt.subplot(1,3,2))\n",
    "cax = plt.scatter(mean_act_states,mean_r_states, c = np.log(mean_energy_states)/np.log(10),alpha = 0.2, cmap = \"autumn\"  )\n",
    "cbar = fig.colorbar(cax) \n",
    "\n",
    "plt.title(\"Random States from trajectory\")\n",
    "plt.xlabel(\"mean act u\")\n",
    "plt.ylabel(\"mean act r\")\n",
    "\n",
    "mean_act_states = []\n",
    "mean_energy_states  = []\n",
    "mean_r_states = []\n",
    "\n",
    "for i, state in enumerate(random_states):\n",
    "  inputs = np.zeros((1,27))\n",
    "  input_val = np.argmax(x[i])\n",
    "  inputs[0,input_val]=1\n",
    "  state  = np.reshape( np.random.uniform(-1, 1, num_nodes), [1, -1])\n",
    "  tensors = [pre_activations[\"fg_bias\"], pre_activations[\"xh_t_bxipn\"],pre_activations[\"r_act_bxn\"],pre_activations[\"u_act_bxn\"],pre_activations[\"xrh_t_bxipn\"], pre_activations[\"c_act_bxn\"]]\n",
    "  tensors_eval = sess.run(tensors,  {input_placeholder:inputs, state_placeholder: state})\n",
    "  mean_act_states.append((sigmoid(tensors_eval[3][0])))\n",
    "  mean_r_states.append((sigmoid(tensors_eval[2][0])))\n",
    "  energy_new, _ = get_energy( state, input_val, componentwise = True)\n",
    "  mean_energy_states.append(energy_new)\n",
    "  \n",
    "plots.append(plt.subplot(1,3,3))\n",
    "cax = plt.scatter(mean_act_states,mean_r_states, c = np.log(mean_energy_states)/np.log(10),alpha = 0.2, cmap = \"autumn\"  )\n",
    "cbar = fig.colorbar(cax) \n",
    "\n",
    "plt.title(\"Random States from all space\")\n",
    "plt.xlabel(\"mean act u\")\n",
    "plt.ylabel(\"mean act r\")\n",
    "for i, plot in enumerate(plots):\n",
    "  plot.set_xlim([0,1])\n",
    "  plot.set_ylim([0,1])\n",
    "\n",
    "  \n",
    "  \n",
    "#Should add one more plot: Random staets from space\n",
    "#How does speed increase as you start sampling from the inference path? \n",
    "#How does does speed relate to x-entropy\n",
    "#How does speed increase as a function of radius away from the inference path and from random point? \n",
    "#How does Taylor expansion error increase as function of radius away from inference path and from random point?\n",
    "#For given radius, what's the correlation between taylor expansion error and speed of the random point\n",
    "#How does speed developt with randomly shuffled dimensions along the inference path\n",
    "#What is the volume of the manifold that the inference path lies on? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "t0-cNoZPzpCn"
   },
   "outputs": [],
   "source": [
    "# This measure speed of random samples in the space of all points.\n",
    "energies_random = []\n",
    "coer_random = []\n",
    "for i in range(10000):\n",
    "  state_in = np.reshape( np.random.uniform(-1, 1, num_nodes), [1, -1])\n",
    "  input_val = np.argmax(x[index[i]])\n",
    "  energy, coer = get_energy(state_in, input_val)\n",
    "  energies_random.append(energy)\n",
    "  coer_random.append(coer)\n",
    "  if i % 2000 == 0:\n",
    "    print( i )\n",
    "    \n",
    "energies_sampled = []\n",
    "coer_sampled = []\n",
    "#h  = np.zeros((1, num_nodes))\n",
    "for i in range(10000):\n",
    "  state_in = random_states[index[i]:index[i]+1,:]\n",
    "  input_val = np.argmax(x[index[i]])\n",
    "  energy, coer = get_energy(state_in, input_val)\n",
    "  energies_sampled.append(energy) \n",
    "  coer_sampled.append(coer)\n",
    "  if i % 2000 == 0:\n",
    "    print( i )\n",
    "  \n",
    "axs = []\n",
    "bins_energy =get_bins([np.log(energies_random)/np.log(10), np.log(energies_sampled)/np.log(10)],100)\n",
    "axs.append( plt.subplot(1,2,1))\n",
    "axs.append( plt.subplot(1,2,2))\n",
    "p1 = axs[0].hist(np.log(energies_random) / np.log(10), bins=bins_energy)\n",
    "p2 = axs[0].hist(np.log(energies_sampled) / np.log(10), bins=bins_energy)\n",
    "\n",
    "bins_coer =get_bins([coer_random, coer_sampled],100)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "XWetYaxm2gOg"
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(16, 7))\n",
    "\n",
    "axs = []\n",
    "\n",
    "val_random = np.log(energies_random)/np.log(10)\n",
    "val_sampled = np.log(energies_sampled)/np.log(10)\n",
    "\n",
    "# val_random = energies_random\n",
    "# val_sampled = energies_sampled\n",
    "\n",
    "bins_energy =get_bins([val_random, val_sampled],100)\n",
    "axs.append( plt.subplot(1,2,1))\n",
    "axs.append( plt.subplot(1,2,2))\n",
    "p1 = axs[0].hist(val_random, bins=bins_energy)\n",
    "p2 = axs[0].hist(val_sampled, bins=bins_energy,alpha = 0.7, color ='g')\n",
    "\n",
    "bins_coer =get_bins([coer_random, coer_sampled],100)\n",
    "\n",
    "p1 = axs[1].hist(coer_random, bins=bins_coer)\n",
    "p2 = axs[1].hist(coer_sampled, bins=bins_coer,alpha = 0.7, color ='g' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "6P834xlnMwRU"
   },
   "outputs": [],
   "source": [
    "# This measure speed of random samples in the space of all points.\n",
    "#energies_random = []\n",
    "#coer_random = []\n",
    "for i in range(1000000):\n",
    "  state_in = np.reshape( np.random.uniform(-1, 1, num_nodes), [1, -1])\n",
    "  input_val = np.argmax(x[index[i%10000]])\n",
    "  energy, coer = get_energy(state_in, input_val)\n",
    "  energies_random.append(energy)\n",
    "  coer_random.append(coer)\n",
    "  if i % 50000 == 0:\n",
    "    print( i )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "qLZ4RrwS7jMj"
   },
   "outputs": [],
   "source": [
    "def get_bins(arrays, n):\n",
    "  curr_min = np.min(arrays[0])\n",
    "  curr_max = np.max(arrays[0])\n",
    "  for a in arrays:\n",
    "    curr_min = np.min([np.min(a), curr_min])\n",
    "    curr_max = np.max([np.max(a), curr_max])\n",
    "  return np.arange(curr_min, curr_max, (curr_max - curr_min) / n)\n",
    "                   \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "5_k-zBmE-pUJ"
   },
   "outputs": [],
   "source": [
    "plt.scatter(energies_random, coer_random, alpha = 0.4)\n",
    "plt.scatter(energies_sampled, coer_sampled, c = 'r', hold = True, alpha = 0.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "CMX-mDsaeRxL"
   },
   "outputs": [],
   "source": [
    "#rest of notebook is random stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "2oa6NsIa1SJM"
   },
   "outputs": [],
   "source": [
    "#tytyt\n",
    "#error with proximity penalty\n",
    "sess.run(approx_error_weight.assign([0]))\n",
    "sess.run(fp_penalty_weight.assign([1]))\n",
    "sess.run(proximity_penalty_weight.assign([0.001]))\n",
    "\n",
    "batch_size_fp = 10 \n",
    "#start the distance by energy plot\n",
    "df_expansions_prox = pd.DataFrame()\n",
    "#num_trials = 1\n",
    "num_trials = 1 \n",
    "for k in range(num_trials):\n",
    "  np.random.shuffle(index)\n",
    "\n",
    "  starting_states = random_states[index[:batch_size_fp*27],:] + np.random.randn(batch_size_fp*27,num_nodes)*0.00001\n",
    "  print starting_states.shape\n",
    "  _ = sess.run( fixed_point_rnn_cell.expansion_points.assign( np.reshape(starting_states, [batch_size_fp, 27, -1])))\n",
    "\n",
    "  constant_input = np.zeros((batch_size_fp * 27,27))\n",
    "  constant_input[:,char] = 1\n",
    "  bounds = np.ones(batch_size_fp * 27)\n",
    "  #First use gradients\n",
    "  old_loss = 1000\n",
    "  for i in range(1000):\n",
    "    _, loss_val, error_in_dim, expansion_points_eval  = sess.run( [optimizer_2, loss_2, outputs_t, fixed_point_rnn_cell.expansion_points] , feed_dict={input_placeholder:constant_input, state_placeholder:starting_states })\n",
    "    expansion_points_eval = np.reshape(expansion_points_eval, [batch_size_fp * 27,-1])\n",
    "    error_in_dim = np.sum(error_in_dim**2,1)\n",
    "    if i % 100 == 0:\n",
    "      print i\n",
    "      for j in range(len( error_in_dim)): \n",
    "        energy = error_in_dim[j]\n",
    "        dist = np.sqrt(np.sum((starting_states[j,:] - expansion_points_eval[j,:])**2))\n",
    "        d = {\"input\" : char, \"state\": expansion_points_eval[j], \"energy\":energy, \"dist\":dist, \"step\":i}\n",
    "        df_expansions_prox = df_expansions_prox.append(d, ignore_index=True)\n",
    "    if i % 1000 == 0:\n",
    "      print i,loss_val, np.mean(error_in_dim), np.mean(df_expansions_prox[\"dist\"])\n",
    "  #df_expansions.to_pickle(\"/tmp/results_csv_huge_fp_new.pkl\") \n",
    "  print \"len(df_expansions_prox): {}\".format(len(df_expansions_prox))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "biuudqhUbKwH"
   },
   "outputs": [],
   "source": [
    "filtered= df_expansions_prox[\"step\"]>0\n",
    "plt.plot(df_expansions_prox[filtered][\"dist\"].values, df_expansions_prox[filtered][\"energy\"].values,'r.')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "9ni1SQNMCs-r"
   },
   "outputs": [],
   "source": [
    "constant_input = np.zeros((batch_size_fp * 27,27))\n",
    "for i in range(270):\n",
    "  constant_input[:,i%27] = 1\n",
    "df_expansions = pd.DataFrame()\n",
    "num_trials = 10000\n",
    "for k in range(num_trials):\n",
    "  np.random.shuffle(index)\n",
    "\n",
    "  starting_states = random_states[index[:batch_size_fp*27],:] + np.random.randn(batch_size_fp*27,num_nodes)*0.1*(k/num_trials)\n",
    "  print starting_states.shape\n",
    "  _ = sess.run( fixed_point_rnn_cell.expansion_points.assign( np.reshape(starting_states, [batch_size_fp, 27, -1])))\n",
    "\n",
    "  bounds = np.ones(batch_size_fp * 27)\n",
    "  #First use gradients\n",
    "  old_loss = 1000\n",
    "  for i in range(20000):\n",
    "    _, loss_val, error_in_dim, expansion_points_eval  = sess.run( [graph_dict['optimizer'],graph_dict['loss'], outputs_t, fixed_point_rnn_cell.expansion_points] , feed_dict={input_placeholder:constant_input})\n",
    "    expansion_points_eval = np.reshape(expansion_points_eval, [batch_size_fp * 27,-1])\n",
    "    error_in_dim = np.sum(error_in_dim**2,1)\n",
    "    if i % 10 == 0:\n",
    "      for j, val in enumerate( error_in_dim): \n",
    "        if val < bounds[ j ]:\n",
    "          bounds[ j ] = bounds[ j ] /10\n",
    "          dist = np.sqrt(np.sum((expansion_points_eval[j] - starting_states[j])**2))\n",
    "          d = {\"input\" : j%27, \"state\": expansion_points_eval[j], \"energy\":val, \"fake\" : 0, \"starting_state\":starting_states[j], \"step\":i, \"dist\":dist}\n",
    "          df_expansions = df_expansions.append(d, ignore_index=True)\n",
    "      if np.abs(old_loss - loss_val) < .00001: #Note, I lowered this value\n",
    "        print \"converged to loss {}\".format(loss_val)\n",
    "        print d\n",
    "        break\n",
    "      old_loss = loss_val\n",
    "    if i % 1000 == 0:\n",
    "      print i,loss_val, np.mean(error_in_dim)\n",
    "  #df_expansions.to_pickle(\"/tmp/results_csv_huge_fp_new\" + FLAGS.source_rnn + \".pkl\") \n",
    "  print \"len(df_expansion): {}\".format(len(df_expansions))\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "kl_1CT2hph2j"
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(10, 7))\n",
    "df_tmp = final_df[( final_df[\"fake\" ] == 0 )]\n",
    "plots = []\n",
    "plots.append(plt.subplot(2,2,1))\n",
    "cax = plt.scatter(df_tmp[\"euc_dist\"], df_tmp[ \"cos_dist\"], c = [np.log(df_tmp[ \"energy\"].iloc[i]) for i in range(len(df_tmp[ \"energy\"]))], alpha = 0.05, cmap = 'autumn' )\n",
    "plt.title(\"cos_dist FP\")\n",
    "cbar = fig.colorbar(cax)\n",
    "\n",
    "\n",
    "plots.append(plt.subplot(2,2,3))\n",
    "cax = plt.scatter(df_tmp[\"euc_dist\"], df_tmp[ \"err\"], c =[np.log(df_tmp[ \"energy\"].iloc[i]) for i in range(len(df_tmp[ \"energy\"]))], alpha = 0.05, cmap = 'autumn' )\n",
    "plt.title(\"euc_dist FP\")\n",
    "cbar = fig.colorbar(cax)\n",
    "\n",
    "df_tmp = final_df[( final_df[\"fake\" ] == 1 )]\n",
    "plots.append(plt.subplot(2,2,2))\n",
    "cax = plt.scatter(df_tmp[\"euc_dist\"], df_tmp[ \"cos_dist\"], c =[np.log(df_tmp[ \"energy\"].iloc[i]) for i in range(len(df_tmp[ \"energy\"]))], alpha = 0.05, cmap = 'autumn' )\n",
    "cbar = fig.colorbar(cax)\n",
    "\n",
    "plt.title(\"cos_dist RAND\")\n",
    "\n",
    "\n",
    "plots.append(plt.subplot(2,2,4))\n",
    "cax =  plt.scatter(df_tmp[\"euc_dist\"], df_tmp[ \"err\"], c =[np.log(df_tmp[ \"energy\"].iloc[i]) for i in range(len(df_tmp[ \"energy\"]))], alpha = 0.05, cmap = 'autumn')\n",
    "plt.title(\"euc_dist RAND\")\n",
    "cbar = fig.colorbar(cax)\n",
    "\n",
    "#plt.savefig(path_res + \"energy_and_fit_rnn_\" +  FLAGS.source_rnn + \"_num_\" + str(num) + \"_reg_\" + reg + snapshop +  \".pdf\")\n",
    "\n",
    "for i, plot in enumerate(plots):\n",
    "  plot.set_xlim([0,20])\n",
    "  plot.set_ylim([0,(1 - i % 2)*1.4 + ((i % 2)) * 0.14])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "ZfI8IqzJAt78"
   },
   "outputs": [],
   "source": [
    "#WE DONT HAVE A RESULT! FIXED POINTS DONT WORK!! LOOK AT BELOW: BLUE IS RANDOM, RED IS FP!!!  :( :( :( \n",
    "fig = plt.figure(figsize=(14, 7))\n",
    "\n",
    "df_tmp = final_df\n",
    "grouped = df_tmp.groupby([ 'fake', 'index'])\n",
    "df_tmp['min_cos'] = grouped['cos_dist'].transform('min')\n",
    "df_tmp['min_err'] = grouped['err'].transform('min')\n",
    "\n",
    "plt.subplot(1,2,1)\n",
    "plt.scatter(df_tmp[df_tmp[\"fake\"] == 0][\"euc_dist\"], df_tmp[df_tmp[\"fake\"] == 0][ \"min_cos\"], c ='r' ,hold=True,  alpha=0.05)\n",
    "plt.scatter(df_tmp[df_tmp[\"fake\"] == 1][\"euc_dist\"], df_tmp[df_tmp[\"fake\"] == 1][ \"min_cos\"],  c='b' ,hold=True,  alpha=0.05)\n",
    "\n",
    "plt.xlabel(\"euc-distance\")\n",
    "plt.ylabel(\"cosine distance\")\n",
    "\n",
    "\n",
    "plt.title(\"cos distance\")\n",
    "\n",
    "plt.subplot(1,2,2)\n",
    "plt.scatter(df_tmp[df_tmp[\"fake\"] == 0][\"euc_dist\"], df_tmp[df_tmp[\"fake\"] == 0][ \"min_err\"], c ='r' ,hold=True,  alpha=0.05)\n",
    "plt.scatter(df_tmp[df_tmp[\"fake\"] == 1][\"euc_dist\"], df_tmp[df_tmp[\"fake\"] == 1][ \"min_err\"],  c='b' ,hold=True,  alpha=0.05)\n",
    "\n",
    "plt.title(\"err\")\n",
    "\n",
    "plt.xlabel(\"euc-distance\")\n",
    "\n",
    "plt.ylabel(\"rmsq error\")\n",
    "\n",
    "\n",
    "\n",
    "df_tmp.groupby([ 'fake'])['min_cos', 'min_err'].mean()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "xADIkMz0o3ij"
   },
   "outputs": [],
   "source": [
    "#loads the vanilla rnn model\n",
    "\n",
    "#flags.DEFINE_integer(\"num_fp\", 10, \"\"\"Number of fixed points to run in parallel per input.\"\"\")\n",
    "tf.reset_default_graph()\n",
    "FLAGS.mode = 4\n",
    "#FLAGS = flags.FLAGS\n",
    "FLAGS.source_rnn = \"rnn\"\n",
    "FLAGS.crazy_distance = False\n",
    "FLAGS.num_fp = 10\n",
    "FLAGS.num_fp_used = 5 #Warning: I set this to 5 for now to speed things up.\n",
    "input_size = output_size = 27\n",
    "reg = \"0\"\n",
    "num = 40000\n",
    "# root = \"jfoerster21\"\n",
    "# new_nodes = 40\n",
    "\n",
    "root = \"jfoersternew22\"\n",
    "\n",
    "FLAGS.source_rnn = \"rnn\"\n",
    "#FLAGS.source_rnn = \"gru\"\n",
    "#rnn = FLAGS.source_rnn\n",
    "rnn = \"diff_rnn\"  #WARNING !!!If you want to use the diff_rnn, uncomment this..!\n",
    "rnn = \"fasternn\"\n",
    "hpvalues = {}\n",
    "\n",
    "batch_size = 1\n",
    "\n",
    "rnn_cell_class = rnn_cell_variants.get_rnn_cell_class(rnn)\n",
    "num_nodes, _ = rnn_cell_class.max_num_params_to_num_nodes(input_size, output_size, num, hpvalues, 1)\n",
    "\n",
    "hparams= tf.HParams()\n",
    "hparams.add_hparam('batch_size', batch_size)\n",
    "\n",
    "rnn_cell_class.add_default_params(hparams)\n",
    "tf.Variable(tf.zeros([batch_size, num_nodes]),\n",
    "                          trainable=False)\n",
    "\n",
    "rnn_cell = rnn_cell_class(input_size, num_nodes, output_size, hparams, layer = 'only')\n",
    "\n",
    "input_placeholder = tf.placeholder(tf.float32, shape=[None, input_size])\n",
    "state_placeholder = tf.placeholder(tf.float32, shape=[None, num_nodes])\n",
    "\n",
    "output, hidden_state = rnn_cell(input_placeholder, state_placeholder)\n",
    "#probs = tf.nn.softmax(output) #\n",
    "\n",
    "#rnn_saver =  tf.train.Saver(tf.trainable_variables() )\n",
    "for vr in tf.trainable_variables():\n",
    "  shp = vr.get_shape()\n",
    "  n_vr = int(np.prod(shp))\n",
    "  print \"param %s nels %d shape \"%(vr.name, n_vr), shp\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "# path = \"/cns/ok-d/home/jfoerster/rnn_tune/\" + root + \"__rv\" + FLAGS.source_rnn + \"__tttext8__np\"+str(num)+ \\\n",
    "#        \"__n1_source\" + FLAGS.source_rnn + \"_l2_reg1/experiments_rnnfixed_point\"\n",
    "path = \"/cns/ok-d/home/jfoerster/rnn_tune/jfoerster23__rv\" + FLAGS.source_rnn + \"__tttext8__np\" + str(num) +\"__n1_source\" +FLAGS.source_rnn + \"_l2_reg\" + reg + \"/experiments0_all\" #This is a new run with simplified logic for hidden state\n",
    "\n",
    "# path = \"/cns/ok-d/home/jfoerster/rnn_tune/\" + root + \"rvdiff_rnn__tttext8__np\"+str(num)+\"__n1_sourcernn_l2_reg1/experiments_secondary\" + str(FLAGS.mode)+\"_all\"\n",
    "  \n",
    "#rnn_weights_path = \"/cns/ok-d/home/jfoerster/rnn_tune/jfoerster21__rv\" + rnn + \"__tttext8__np\" + str(num) + \"__n1_source\" + rnn +  \"_l2_reg0.1/experiments_rnndiff_rnn\"\n",
    "\n",
    "print path\n",
    "rnn_saver.restore( sess, path)\n",
    "fixed_points = sess.run(rnn_cell.expansion_points)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "E5ARNwEvwjRI"
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def get_min_dist(fps, h):\n",
    "  min_dist = (np.sum(fps[0] - h_in)**2)**0.5\n",
    "  indx = 0\n",
    "  for ind, fp in enumerate( fps ):\n",
    "    if ind < FLAGS.num_fp_used:\n",
    "      d = (np.sum(fp - h_in)**2)**0.5\n",
    "      if d < min_dist:\n",
    "        min_dist = d\n",
    "        indx = ind\n",
    "  return min_dist, indx\n",
    "\n",
    "task_class, _ = rnn_tuner.get_training_task_class(\"text8\")\n",
    "train_task = task_class(\"\", batch_size, 100000,None, state_seed=1234, noise_seed=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "6ofLVghZrsaa"
   },
   "outputs": [],
   "source": [
    "#Compare quality of fit for random hidden states to random fixed points. Plot comparison by speed\n",
    "input_placeholder = tf.placeholder(tf.float32, shape=[None, input_size])\n",
    "state_placeholder = tf.placeholder(tf.float32, shape=[None, num_nodes])\n",
    "\n",
    "output, hidden_state = rnn_cell(input_placeholder, state_placeholder)\n",
    "probs = tf.nn.softmax(output)\n",
    "\n",
    "print path\n",
    "rnn_saver.restore( sess, path)\n",
    "\n",
    "_= sess.run(rnn_cell.expansion_points.assign(fixed_points))\n",
    "\n",
    "x, y = train_task.get_valid_next_batch()\n",
    "print len(x), len(y)\n",
    "print x[1].shape\n",
    "\n",
    "h = np.zeros([batch_size, num_nodes])\n",
    "#h  = sess.run(rnn_cell.zero_state)\n",
    "df_fp = pd.DataFrame()\n",
    "errors = []\n",
    "for i in xrange(300):\n",
    "  if i % 100 == 0:\n",
    "    print i\n",
    "  h_in = h\n",
    "  t, h = sess.run([output, hidden_state], feed_dict={input_placeholder:x[i], state_placeholder:h})\n",
    "  mse = np.mean(t**2)\n",
    "  errors.append(t**2)\n",
    "  #fixed_to_h = np.mean((fixed_points[0, np.argmax(x[i])] - h_in)**2)\n",
    "  min_dist, indx = get_min_dist(fixed_points[:,np.argmax(x[i]), :], h_in)\n",
    "  d = {\"step\":i, \"diff\":t, \"h\":h, \"x\":x[i], \"y\":y[i], \"dist\" :min_dist, \"mse\":mse, \"ind\" :indx}\n",
    "  df_fp = df_fp.append(d, ignore_index=True)\n",
    "\n",
    "print np.mean(errors)\n",
    "\n",
    "#Pick random hidden states\n",
    "copy_h = df[\"h\"].values[:]\n",
    "copy_h = [copy_h[i][0] for i in xrange(len(copy_h))]\n",
    "copy_h = np.array(copy_h)\n",
    "\n",
    "np.random.shuffle(copy_h)\n",
    "random_h = copy_h[:270]\n",
    "random_h = np.reshape(random_h, [10, 27, -1])\n",
    "print random_h.shape\n",
    "\n",
    "#Assign hidden states to the expansion point\n",
    "_= sess.run(rnn_cell.expansion_points.assign(random_h))\n",
    "\n",
    "h = np.zeros([batch_size, num_nodes]) \n",
    "df = pd.DataFrame()\n",
    "errors = []\n",
    "for i in xrange(300):\n",
    "  if i % 100 == 0:\n",
    "    print i\n",
    "  h_in = h\n",
    "  t, h = sess.run([output, hidden_state], feed_dict={input_placeholder:x[i], state_placeholder:h})\n",
    "  mse = np.mean(t**2)\n",
    "  errors.append(t**2)\n",
    "  min_dist, indx = get_min_dist(random_h[:,np.argmax(x[i]), :], h_in)\n",
    "  d = {\"step\":i, \"diff\":t, \"h\":h, \"x\":x[i], \"y\":y[i], \"dist\" :min_dist, \"mse\":mse, \"ind\" : indx}\n",
    "  df = df.append(d, ignore_index=True)\n",
    "\n",
    "print np.mean(errors)\n",
    "\n",
    "inputs = random_h\n",
    "#inputs = fixed_points\n",
    "mode = 4\n",
    "#fixed_points = sess.run(rnn_cell.expansion_points)\n",
    "avgs = []\n",
    "count = 0\n",
    "for fixed_idx in xrange(len(fixed_points)):\n",
    "  if fixed_idx != 0 and False:\n",
    "    continue\n",
    "  for char_idx in xrange(len(fixed_points[0])):\n",
    "    fp = np.array([inputs[fixed_idx][char_idx]])\n",
    "    char = np.array([np.eye(27)[char_idx]])\n",
    "    #random.shuffle(fp[0])\n",
    "    #fp = random_h[count]\n",
    "    count+=1\n",
    "    fp_image = sess.run([hidden_state], feed_dict={input_placeholder:char, state_placeholder:fp})\n",
    "    avg = np.mean(np.abs(fp_image - fp))\n",
    "    #print fixed_idx, char_idx, avg\n",
    "    avgs.append(avg)\n",
    "    \n",
    "speed =  []\n",
    "for i in ( df[\"ind\"]):\n",
    "  speed.append(avgs[np.int32(i)])\n",
    "  \n",
    "fig = plt.figure(figsize=(15, 7))\n",
    "plt.subplot(1, 2, 1)\n",
    "plt.scatter(df[\"dist\"], df[\"mse\"],  c = speed)\n",
    "plt.title(\"random points\")\n",
    "print( np.corrcoef(df[\"dist\"], df[\"mse\"]))\n",
    "\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.plot(df_fp[\"dist\"], df_fp[\"mse\"], \"r.\")\n",
    "plt.title(\"fixed points\")\n",
    "print( np.corrcoef(df_fp[\"dist\"], df_fp[\"mse\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "DoDM4dSesnnZ"
   },
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "def plot_tsne(data_points, labels):\n",
    "  \n",
    "  model = TSNE(n_components=2, random_state=0)\n",
    "  np.set_printoptions(suppress=True)\n",
    "  fit_tsne = model.fit_transform(data_points)\n",
    "\n",
    "  def map_vals(vals):\n",
    "    vals_set = list(set(vals))\n",
    "    map_dict = {}\n",
    "    for i, v in enumerate(vals_set):\n",
    "      map_dict[v] = i\n",
    "    return [map_dict[v] for v in vals]\n",
    "\n",
    "  print fit_tsne\n",
    "\n",
    "  x = [h[0] for h in h_tnse]\n",
    "  y = [h[1] for h in h_tnse]\n",
    "\n",
    "  colors = map_vals(labels)\n",
    "  print len(set(colors)) \n",
    "\n",
    "  p = plt.scatter(x,y,c = colors, cmap = \"Set1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "WNqo_ITT6A1k"
   },
   "outputs": [],
   "source": [
    "print w_out.shape\n",
    "print w_x.shape\n",
    "print b_ixn.shape\n",
    "print out_bias.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "QsLb24u3IJ4q"
   },
   "outputs": [],
   "source": [
    "np.corrcoef(np.zeros((10,100)), rowvar=False).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "OoJkg6J89CJR"
   },
   "outputs": [],
   "source": [
    "top_10_ev = []\n",
    "top_10_el = []\n",
    "\n",
    "top_1_v = []\n",
    "\n",
    "top_10_v = []\n",
    "top_10_u = []\n",
    "\n",
    "plt.figure(figsize=(15,10), dpi=300)\n",
    "for i,c in enumerate(mapping):\n",
    "  plt.subplot(5,6,i+1)\n",
    "  char = mapping[c]\n",
    "\n",
    "  el, ev = np.linalg.eig(w_x[char])\n",
    "  top_10_ev.append(ev[:, :10])\n",
    "  top_10_el.append(el[:10])\n",
    "  \n",
    "  u,s,v = np.linalg.svd(w_x[char])\n",
    "  \n",
    "  top_1_v.append(v[:1,:].T)\n",
    "  top_10_v.append(v[:10, :].T)\n",
    "  top_10_u.append(u[:,:10])\n",
    "  \n",
    "  #plt.imshow(np.abs(np.corrcoef(np.abs(v[:, :50]), rowvar=False,)), \n",
    "  #           'Greys_r', interpolation='nearest')\n",
    "  #plt.colorbar()\n",
    "  plt.stem(s)\n",
    "  \n",
    "  \n",
    "  plt.title(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "uk-6XxiTX3eJ"
   },
   "outputs": [],
   "source": [
    "np.linalg.svd?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "2vHW8NQwYSm3"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "VOqQQljZWY_j"
   },
   "outputs": [],
   "source": [
    "plt.imshow(cd(v[:10,: ].T, w_out), interpolation='nearest')\n",
    "plt.grid(False)\n",
    "plt.colorbar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "g5Cn2c7-ovGM"
   },
   "outputs": [],
   "source": [
    "def cd(u,v):\n",
    "  u = u / np.sqrt((u**2).sum(0, keepdims=True))\n",
    "  v = v / np.sqrt((v**2).sum(0, keepdims=True))\n",
    "  return 1-u.T.dot(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "NB08kxuXTItV"
   },
   "outputs": [],
   "source": [
    "np.max(np.abs((w_x[char] - u.dot(np.diag(s)).dot(v))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "-T8JGrNOlbVR"
   },
   "outputs": [],
   "source": [
    "spc_fix_pt = np.linalg.inv(np.eye(216) - w_x[mapping[' ']]).dot(b_ixn[mapping[' ']])\n",
    "spc_fix_pt = b_ixn[mapping[' ']]\n",
    "plt.scatter(spc_fix_pt, top_10_v[1][:,0])\n",
    "\n",
    "np.corrcoef(spc_fix_pt, top_10_v[1][:,0])\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "N0KalFmymDy1"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(15,15))\n",
    "\n",
    "k = 5\n",
    "top_10_v_s = np.hstack([m[:,:k] for m in top_10_v])\n",
    "print top_10_v_s.shape\n",
    "top_10_u_s = np.hstack([m[:,:k] for m in top_10_u])\n",
    "print top_10_u_s.shape\n",
    "\n",
    "vecs = np.hstack((top_10_v_s, top_10_u_s, b_ixn.T, w_out))\n",
    "vecs.shape\n",
    "\n",
    "plt.imshow(np.abs(1 - cd(vecs,vecs)), 'Greys_r', interpolation='nearest')\n",
    "plt.colorbar()\n",
    "\n",
    "plt.yticks(np.arange(k*0.5 - 0.5, 27*k ,k), [pretty_chars[i] for i in xrange(27)])\n",
    "plt.xticks(np.arange(k*0.5 - 0.5, 27*k ,k), [pretty_chars[i] for i in xrange(27)])\n",
    "plt.grid(False)\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "IOs9Qa_OuhWi"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "x3NJHl41Px6z"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10,10))\n",
    "\n",
    "k = 1\n",
    "top_10_v_s = np.hstack([m[:,:k] for m in top_10_v])\n",
    "print top_10_v_s.shape\n",
    "top_10_u_s = np.hstack([m[:,:k] for m in top_10_u])\n",
    "print top_10_u_s.shape\n",
    "\n",
    "plt.imshow(np.abs(np.corrcoef(np.hstack((top_10_v_s, top_10_u_s, b_ixn.T, w_out)), rowvar=False)), 'Greys_r', interpolation='nearest')\n",
    "plt.colorbar()\n",
    "\n",
    "plt.yticks(np.arange(k*0.5 - 1, 27*k ,k), [pretty_chars[i] for i in xrange(27)])\n",
    "plt.xticks(np.arange(k*0.5 - 1, 27*k ,k), [pretty_chars[i] for i in xrange(27)])\n",
    "plt.grid(False)\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "rdRq3OET_It9"
   },
   "outputs": [],
   "source": [
    "plt.plot(np.abs(ev[:,:10].T.dot(w_out).T))\n",
    "plt.vlines(char, *plt.ylim())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "cj5zHjK99EuP"
   },
   "outputs": [],
   "source": [
    "top_10_ev = []\n",
    "top_10_el = []\n",
    "\n",
    "top_1_v = []\n",
    "\n",
    "top_10_v = []\n",
    "top_10_u = []\n",
    "\n",
    "plt.figure(figsize=(15,10), dpi=300)\n",
    "for i,c in enumerate(mapping):\n",
    "  plt.subplot(5,6,i+1)\n",
    "  char = mapping[c]\n",
    "\n",
    "  el, ev = np.linalg.eig(w_x[char])\n",
    "  top_10_ev.append(ev[:, :10])\n",
    "  top_10_el.append(el[:10])\n",
    "  \n",
    "  u,s,v = np.linalg.svd(w_x[char])\n",
    "  \n",
    "  top_1_v.append(v[:,:1])\n",
    "  top_10_v.append(v[:,:10])\n",
    "  top_10_u.append(u[:,:10])\n",
    "  \n",
    "  #plt.imshow(np.abs(np.corrcoef(np.abs(v[:, :50]), rowvar=False,)), \n",
    "  #           'Greys_r', interpolation='nearest')\n",
    "  #plt.colorbar()\n",
    "  plt.stem(s)\n",
    "  \n",
    "  \n",
    "  plt.title(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "BCYNCjI5O2Om"
   },
   "outputs": [],
   "source": [
    "string = \" the long and winding text and some more of it improbable \"\n",
    "\n",
    "hs = [h_zero.T]\n",
    "\n",
    "for char in string:\n",
    "  hs.append(make_step(hs[-1], mapping[char])[0])\n",
    "\n",
    "hs = np.hstack(hs[1:])\n",
    "\n",
    "kk=1\n",
    "V = np.hstack([m[:,:1] for m in top_10_v[kk:kk+1]])\n",
    "U = np.hstack([m[:,:1] for m in top_10_u[:]])\n",
    "\n",
    "if 0:\n",
    "  plt.imshow(V.T.dot(hs), interpolation='nearest', cmap='jet')\n",
    "  plt.xticks(*zip(*enumerate(string)))\n",
    "  plt.grid(False)\n",
    "  plt.axis('tight')\n",
    "  \n",
    "#plt.plot(V.T.dot(hs)[0,:])\n",
    "#plt.plot((w_out.T.dot(hs) + out_bias[:,None])[0]*-0.13)\n",
    "\n",
    "plt.scatter(V.T.dot(hs), (w_out.T.dot(hs) + out_bias[:,None])[0])\n",
    "\n",
    "#plt.xticks(*zip(*enumerate(string)))\n",
    "#plt.grid(False)\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "GIFXXqrCarYO"
   },
   "outputs": [],
   "source": [
    "plt.plot(U[:,:2].T.dot(w_out).T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "C52n2Q2YUwX0"
   },
   "outputs": [],
   "source": [
    "plt.plot(U[:,:2].T.dot(w_out).T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "iOAH7aGG_jZf"
   },
   "outputs": [],
   "source": [
    "-\n",
    "\n",
    "u_1, s_1, v_1 = np.linalg.svd( w_x[mapping['t']] )\n",
    "u_2, s_2, v_2 = np.linalg.svd( w_x[mapping['h']] )\n",
    "\n",
    "\n",
    "plt.figure()\n",
    "mmm = v_2.dot(u_1)\n",
    "mmm = np.diag(np.sqrt(s_2)).dot(mmm.dot(np.diag(np.sqrt(s_1))))\n",
    "plt.imshow(np.abs(mmm), 'Greys')\n",
    "plt.grid(False)\n",
    "plt.colorbar()\n",
    "\n",
    "plt.figure()\n",
    "plt.hist((mmm).ravel(), bins=250)\n",
    "#plt.ylim(0, 200)\n",
    "\n",
    "(np.abs(mmm)>1e-1).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "nWwTEe5VsRTW"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "KIo69gq0tqIy"
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": null,
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "W9CWy8kgxFxV"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "Explore_ISAN",
   "provenance": [
    {
     "file_id": "0Byvnfpemwz8qOWVWN2hLbnhfN2s",
     "timestamp": 1478819652519
    },
    {
     "file_id": "0B_HS9zyENJqVTnpCelV1M1BuNDg",
     "timestamp": 1478044437806
    },
    {
     "file_id": "0Byvnfpemwz8qQjFzQ0hQNlZDRGc",
     "timestamp": 1477431048978
    },
    {
     "file_id": "0B_HS9zyENJqVNkw2eW5zdFVPZFE",
     "timestamp": 1477343427690
    },
    {
     "file_id": "0Byvnfpemwz8qWE5yMUNiLTV1eHc",
     "timestamp": 1476813640441
    }
   ],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}