CartPole_Q_learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "import gym\n",
    "from gym import wrappers\n",
    "from gym.error import ResetNeeded\n",
    "\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(4, input_shape=[1, 4], activation='relu', kernel_initializer='random_uniform'))\n",
    "    model.add(Dense(6, activation='relu', kernel_initializer='random_uniform'))\n",
    "    model.add(Dense(2, kernel_initializer='random_uniform'))\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_MODE = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "env = wrappers.Monitor(env, '/tmp/cartpole-experiment-1', force=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "index 1 is out of bounds for axis 1 with size 1",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-69-af78fd10fc71>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     54\u001b[0m             \u001b[0mQ_traget\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m             \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m             \u001b[0mQ_traget\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreward\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mLEARNING_RATE\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mQ_tmp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     57\u001b[0m             \u001b[0mobserv_hist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mQ_traget\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: index 1 is out of bounds for axis 1 with size 1"
     ]
    }
   ],
   "source": [
    "CONFIDENCE_LEVEL = .9\n",
    "LEARNING_RATE = .9\n",
    "STEPS_IN_HIST = 3\n",
    "\n",
    "observ_hist = []\n",
    "reward_hist = []\n",
    "s = env.reset()\n",
    "done = False\n",
    "\n",
    "# if TRAIN_MODE:\n",
    "#     model = get_model()\n",
    "#     model.compile('adam', 'mse')\n",
    "\n",
    "max_angle = 0\n",
    "pos_cicles = 0\n",
    "neg_cicles = 0\n",
    "\n",
    "while True:\n",
    "    try:\n",
    "#         observ_hist.append(s)\n",
    "#         reward_hist.append(0 if done else 1)\n",
    "        \n",
    "        if TRAIN_MODE and np.random.rand() < CONFIDENCE_LEVEL:\n",
    "            Q = random.choice([np.array([1, 0]).reshape((-1, 2)), \n",
    "                               np.array([0, 1]).reshape((-1, 2))])\n",
    "#             print Q.shape\n",
    "        else:\n",
    "            Q = model.predict_proba(s.reshape((-1, 1, 4)), verbose=0)\n",
    "            \n",
    "        \n",
    "        s1, _, done, info = env.step(Q.argmax())\n",
    "#         print(s1)\n",
    "#         reward = getting_reward(s1)\n",
    "        reward = 0\n",
    "        if s1[1] > 0:\n",
    "            pos_cicles += 1\n",
    "            if neg_cicles <= 2:\n",
    "                neg_cicles = 0\n",
    "        else:\n",
    "            neg_cicles += 1\n",
    "            if pos_cicles <= 2:\n",
    "                pos_cicles = 0 \n",
    "\n",
    "        if neg_cicles > 2 and pos_cicles > 2:\n",
    "#             print('agent boosted for balancing attempt')\n",
    "            reward += 1\n",
    "#             reward_boost *= reward_boost_scaler + (10/((neg_cicles + pos_cicles)/2))\n",
    "            print('here')\n",
    "            pos_cicles, neg_cicles = 0, 0\n",
    "        \n",
    "        if TRAIN_MODE:\n",
    "            Q_tmp = model.predict_proba(s1.reshape((-1, 1, 4)))\n",
    "\n",
    "            Q_traget = Q\n",
    "            action = Q.argmax()\n",
    "            Q_traget[0, action] = reward + LEARNING_RATE * Q_tmp.max()\n",
    "            observ_hist.append([s.reshape((-1, 4)),  Q_traget.reshape((-1, 2))])\n",
    "            \n",
    "            model.fit(s.reshape((-1, 1, 4)), Q_traget.reshape((-1, 1, 2)), verbose=0)\n",
    "\n",
    "        s = s1\n",
    "        \n",
    "    except ResetNeeded:\n",
    "        s = env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(observ_hist)\n",
    "\n",
    "o_h, r_h = [], []\n",
    "for observ in observ_hist:\n",
    "    o_h.append(observ[0])\n",
    "    r_h.append(observ[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=np.array(o_h)\n",
    "Y=np.array(r_h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reward_hist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = np.array([10,2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t.argmax()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = get_model()\n",
    "model.compile('adam', 'mse')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(X, Y, epochs=100000, batch_size=128, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}